create_data_index.ipynb

{
 "cells": [
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "The goal is to crawl in lexicographical order over the folders (and, within a folder, a lexicographical crawl over the files) that're under 'true_false_datasets', creating a data structure that implicitly assigns a globally (within-project) unique numeric id to each record in each dataset file.",
   "id": "52d3b4d36f1b9387"
  },
  {
   "cell_type": "code",
   "id": "initial_id",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2024-10-13T19:33:14.584448Z",
     "start_time": "2024-10-13T19:33:14.572221Z"
    }
   },
   "source": [
    "import pandas as pd\n",
    "\n",
    "from phi_3_5_constants import dsets_folder, dsets_index_path"
   ],
   "outputs": [],
   "execution_count": 38
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-13T19:33:14.973554Z",
     "start_time": "2024-10-13T19:33:14.955135Z"
    }
   },
   "cell_type": "code",
   "source": "",
   "id": "75aa32595d0dd842",
   "outputs": [],
   "execution_count": 39
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-13T19:33:15.310761Z",
     "start_time": "2024-10-13T19:33:15.298410Z"
    }
   },
   "cell_type": "code",
   "source": "",
   "id": "33d972af991f80b8",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "true_false_datasets/categ_dataset_pairs.csv\n",
      "true_false_datasets\\animal_class/animal_class.csv\n",
      "true_false_datasets\\animal_class/animal_class_conj.csv\n",
      "true_false_datasets\\animal_class/animal_class_disj.csv\n",
      "true_false_datasets\\animal_class/neg_animal_class.csv\n",
      "true_false_datasets\\cities/cities.csv\n",
      "true_false_datasets\\cities/cities_conj.csv\n",
      "true_false_datasets\\cities/cities_disj.csv\n",
      "true_false_datasets\\cities/neg_cities.csv\n",
      "true_false_datasets\\element_symb/element_symb.csv\n",
      "true_false_datasets\\element_symb/element_symb_conj.csv\n",
      "true_false_datasets\\element_symb/element_symb_disj.csv\n",
      "true_false_datasets\\element_symb/neg_element_symb.csv\n",
      "true_false_datasets\\facts/facts.csv\n",
      "true_false_datasets\\facts/facts_conj.csv\n",
      "true_false_datasets\\facts/facts_disj.csv\n",
      "true_false_datasets\\facts/neg_facts.csv\n",
      "true_false_datasets\\inventors/inventors.csv\n",
      "true_false_datasets\\inventors/inventors_conj.csv\n",
      "true_false_datasets\\inventors/inventors_disj.csv\n",
      "true_false_datasets\\inventors/neg_inventors.csv\n",
      "true_false_datasets\\real_world_scenarios/ambiguous_lie.csv\n",
      "true_false_datasets\\real_world_scenarios/ambiguous_truthful_reply.csv\n",
      "true_false_datasets\\real_world_scenarios/honest_reply_despite_incentive_to_lie.csv\n",
      "true_false_datasets\\real_world_scenarios/unambiguous_lie.csv\n",
      "true_false_datasets\\real_world_scenarios/unambiguous_truthful_reply.csv\n",
      "true_false_datasets\\relative_comparison/larger_than.csv\n",
      "true_false_datasets\\relative_comparison/smaller_than.csv\n",
      "true_false_datasets\\sp_en_trans/neg_sp_en_trans.csv\n",
      "true_false_datasets\\sp_en_trans/sp_en_trans.csv\n",
      "true_false_datasets\\sp_en_trans/sp_en_trans_conj.csv\n",
      "true_false_datasets\\sp_en_trans/sp_en_trans_disj.csv\n",
      "true_false_datasets\\true_false/common_claim_true_false.csv\n",
      "true_false_datasets\\true_false/counterfact_true_false.csv\n"
     ]
    }
   ],
   "execution_count": 40
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-13T17:42:35.192501Z",
     "start_time": "2024-10-13T17:42:35.183635Z"
    }
   },
   "cell_type": "code",
   "source": [
    "dset_folder_file_names: list[tuple[str, str]] = []\n",
    "\n",
    "categ_dirs = sorted([c_dir for c_dir in dsets_folder.iterdir() if c_dir.is_dir()])\n",
    "\n",
    "for categ_dir in categ_dirs:\n",
    "    dset_file_names_in_categ: list[str] = []\n",
    "    for data_file in categ_dir.iterdir():\n",
    "        if data_file.is_dir():\n",
    "            print(f\"WARNING- entry {data_file.name} in folder {categ_dir.name} is a folder rather than the expected data file\")\n",
    "        else:\n",
    "            dset_file_names_in_categ.append(data_file.name)\n",
    "            \n",
    "    dset_file_names_in_categ = sorted(dset_file_names_in_categ)\n",
    "    dset_folder_file_names.extend([(categ_dir.name, file_name_in_categ) for file_name_in_categ in dset_file_names_in_categ])\n",
    "    "
   ],
   "id": "fc3a66e558ef1b26",
   "outputs": [],
   "execution_count": 3
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-13T17:42:35.209040Z",
     "start_time": "2024-10-13T17:42:35.198924Z"
    }
   },
   "cell_type": "code",
   "source": "# dset_folder_file_names",
   "id": "c6bbc4675ceaa914",
   "outputs": [],
   "execution_count": 4
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-13T17:49:50.853159Z",
     "start_time": "2024-10-13T17:49:50.843291Z"
    }
   },
   "cell_type": "code",
   "source": "dset_folder_file_names_df = pd.DataFrame.from_records(dset_folder_file_names, columns=[\"Categ_Folder\", \"Dataset_File\"])",
   "id": "5cecb716445f7655",
   "outputs": [],
   "execution_count": 22
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-13T17:49:51.238150Z",
     "start_time": "2024-10-13T17:49:51.227378Z"
    }
   },
   "cell_type": "code",
   "source": "dset_folder_file_names_df",
   "id": "819520844a71d0c7",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "            Categ_Folder                               Dataset_File\n",
       "0           animal_class                           animal_class.csv\n",
       "1           animal_class                      animal_class_conj.csv\n",
       "2           animal_class                      animal_class_disj.csv\n",
       "3           animal_class                       neg_animal_class.csv\n",
       "4                 cities                                 cities.csv\n",
       "5                 cities                            cities_conj.csv\n",
       "6                 cities                            cities_disj.csv\n",
       "7                 cities                             neg_cities.csv\n",
       "8           element_symb                           element_symb.csv\n",
       "9           element_symb                      element_symb_conj.csv\n",
       "10          element_symb                      element_symb_disj.csv\n",
       "11          element_symb                       neg_element_symb.csv\n",
       "12                 facts                                  facts.csv\n",
       "13                 facts                             facts_conj.csv\n",
       "14                 facts                             facts_disj.csv\n",
       "15                 facts                              neg_facts.csv\n",
       "16             inventors                              inventors.csv\n",
       "17             inventors                         inventors_conj.csv\n",
       "18             inventors                         inventors_disj.csv\n",
       "19             inventors                          neg_inventors.csv\n",
       "20  real_world_scenarios                          ambiguous_lie.csv\n",
       "21  real_world_scenarios               ambiguous_truthful_reply.csv\n",
       "22  real_world_scenarios  honest_reply_despite_incentive_to_lie.csv\n",
       "23  real_world_scenarios                        unambiguous_lie.csv\n",
       "24  real_world_scenarios             unambiguous_truthful_reply.csv\n",
       "25   relative_comparison                            larger_than.csv\n",
       "26   relative_comparison                           smaller_than.csv\n",
       "27           sp_en_trans                        neg_sp_en_trans.csv\n",
       "28           sp_en_trans                            sp_en_trans.csv\n",
       "29           sp_en_trans                       sp_en_trans_conj.csv\n",
       "30           sp_en_trans                       sp_en_trans_disj.csv\n",
       "31            true_false                common_claim_true_false.csv\n",
       "32            true_false                 counterfact_true_false.csv"
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Categ_Folder</th>\n",
       "      <th>Dataset_File</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>animal_class</td>\n",
       "      <td>animal_class.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>animal_class</td>\n",
       "      <td>animal_class_conj.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>animal_class</td>\n",
       "      <td>animal_class_disj.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>animal_class</td>\n",
       "      <td>neg_animal_class.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>cities</td>\n",
       "      <td>cities.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>cities</td>\n",
       "      <td>cities_conj.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>cities</td>\n",
       "      <td>cities_disj.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>cities</td>\n",
       "      <td>neg_cities.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>element_symb</td>\n",
       "      <td>element_symb.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>element_symb</td>\n",
       "      <td>element_symb_conj.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>element_symb</td>\n",
       "      <td>element_symb_disj.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>element_symb</td>\n",
       "      <td>neg_element_symb.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>facts</td>\n",
       "      <td>facts.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>facts</td>\n",
       "      <td>facts_conj.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>facts</td>\n",
       "      <td>facts_disj.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>facts</td>\n",
       "      <td>neg_facts.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>inventors</td>\n",
       "      <td>inventors.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>inventors</td>\n",
       "      <td>inventors_conj.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>inventors</td>\n",
       "      <td>inventors_disj.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>inventors</td>\n",
       "      <td>neg_inventors.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>real_world_scenarios</td>\n",
       "      <td>ambiguous_lie.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>real_world_scenarios</td>\n",
       "      <td>ambiguous_truthful_reply.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>real_world_scenarios</td>\n",
       "      <td>honest_reply_despite_incentive_to_lie.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>real_world_scenarios</td>\n",
       "      <td>unambiguous_lie.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>real_world_scenarios</td>\n",
       "      <td>unambiguous_truthful_reply.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>relative_comparison</td>\n",
       "      <td>larger_than.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>relative_comparison</td>\n",
       "      <td>smaller_than.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>sp_en_trans</td>\n",
       "      <td>neg_sp_en_trans.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>sp_en_trans</td>\n",
       "      <td>sp_en_trans.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>sp_en_trans</td>\n",
       "      <td>sp_en_trans_conj.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>sp_en_trans</td>\n",
       "      <td>sp_en_trans_disj.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>true_false</td>\n",
       "      <td>common_claim_true_false.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>true_false</td>\n",
       "      <td>counterfact_true_false.csv</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 23
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-13T17:49:51.638763Z",
     "start_time": "2024-10-13T17:49:51.633280Z"
    }
   },
   "cell_type": "code",
   "source": "#dset_folder_file_names_df.to_csv(dsets_folder / \"categ_dataset_pairs.csv\", index_label=\"Idx\")",
   "id": "cd0008d3032c6cde",
   "outputs": [],
   "execution_count": 24
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-13T17:49:52.741830Z",
     "start_time": "2024-10-13T17:49:52.723312Z"
    }
   },
   "cell_type": "code",
   "source": [
    "dset_folder_file_names_df['min_global_record_idx'] = -1\n",
    "dset_folder_file_names_df['max_global_record_idx'] = -1"
   ],
   "id": "619c309a34eb11ad",
   "outputs": [],
   "execution_count": 27
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-13T17:50:01.002194Z",
     "start_time": "2024-10-13T17:50:00.994170Z"
    }
   },
   "cell_type": "code",
   "source": "dset_folder_file_names_df.at[10, \"Categ_Folder\"]",
   "id": "816988a3bc9e369d",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'element_symb'"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 29
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-13T17:50:03.571448Z",
     "start_time": "2024-10-13T17:50:03.381265Z"
    }
   },
   "cell_type": "code",
   "source": [
    "num_records_so_far = 0\n",
    "for (idx, categ_folder_nm, dset_file_nm, _, _) in dset_folder_file_names_df.itertuples():\n",
    "    curr_dset_df = pd.read_csv(dsets_folder / categ_folder_nm / dset_file_nm)\n",
    "    dset_folder_file_names_df.at[idx, 'min_global_record_idx'] = num_records_so_far\n",
    "    num_records_so_far += curr_dset_df.shape[0]\n",
    "    dset_folder_file_names_df.at[idx, 'max_global_record_idx'] = num_records_so_far"
   ],
   "id": "e16f722fa6c0ae9f",
   "outputs": [],
   "execution_count": 30
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-13T17:50:04.667809Z",
     "start_time": "2024-10-13T17:50:04.657231Z"
    }
   },
   "cell_type": "code",
   "source": "dset_folder_file_names_df",
   "id": "80693d09207a5b2e",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "            Categ_Folder                               Dataset_File  \\\n",
       "0           animal_class                           animal_class.csv   \n",
       "1           animal_class                      animal_class_conj.csv   \n",
       "2           animal_class                      animal_class_disj.csv   \n",
       "3           animal_class                       neg_animal_class.csv   \n",
       "4                 cities                                 cities.csv   \n",
       "5                 cities                            cities_conj.csv   \n",
       "6                 cities                            cities_disj.csv   \n",
       "7                 cities                             neg_cities.csv   \n",
       "8           element_symb                           element_symb.csv   \n",
       "9           element_symb                      element_symb_conj.csv   \n",
       "10          element_symb                      element_symb_disj.csv   \n",
       "11          element_symb                       neg_element_symb.csv   \n",
       "12                 facts                                  facts.csv   \n",
       "13                 facts                             facts_conj.csv   \n",
       "14                 facts                             facts_disj.csv   \n",
       "15                 facts                              neg_facts.csv   \n",
       "16             inventors                              inventors.csv   \n",
       "17             inventors                         inventors_conj.csv   \n",
       "18             inventors                         inventors_disj.csv   \n",
       "19             inventors                          neg_inventors.csv   \n",
       "20  real_world_scenarios                          ambiguous_lie.csv   \n",
       "21  real_world_scenarios               ambiguous_truthful_reply.csv   \n",
       "22  real_world_scenarios  honest_reply_despite_incentive_to_lie.csv   \n",
       "23  real_world_scenarios                        unambiguous_lie.csv   \n",
       "24  real_world_scenarios             unambiguous_truthful_reply.csv   \n",
       "25   relative_comparison                            larger_than.csv   \n",
       "26   relative_comparison                           smaller_than.csv   \n",
       "27           sp_en_trans                        neg_sp_en_trans.csv   \n",
       "28           sp_en_trans                            sp_en_trans.csv   \n",
       "29           sp_en_trans                       sp_en_trans_conj.csv   \n",
       "30           sp_en_trans                       sp_en_trans_disj.csv   \n",
       "31            true_false                common_claim_true_false.csv   \n",
       "32            true_false                 counterfact_true_false.csv   \n",
       "\n",
       "    min_global_record_idx  max_global_record_idx  \n",
       "0                       0                    164  \n",
       "1                     164                    664  \n",
       "2                     664                   1164  \n",
       "3                    1164                   1328  \n",
       "4                    1328                   2824  \n",
       "5                    2824                   4322  \n",
       "6                    4322                   4822  \n",
       "7                    4822                   6318  \n",
       "8                    6318                   6504  \n",
       "9                    6504                   7004  \n",
       "10                   7004                   7504  \n",
       "11                   7504                   7690  \n",
       "12                   7690                   8251  \n",
       "13                   8251                   8751  \n",
       "14                   8751                   9251  \n",
       "15                   9251                   9812  \n",
       "16                   9812                  10218  \n",
       "17                  10218                  10718  \n",
       "18                  10718                  11218  \n",
       "19                  11218                  11624  \n",
       "20                  11624                  11642  \n",
       "21                  11642                  11665  \n",
       "22                  11665                  11671  \n",
       "23                  11671                  11720  \n",
       "24                  11720                  11775  \n",
       "25                  11775                  13755  \n",
       "26                  13755                  15735  \n",
       "27                  15735                  16089  \n",
       "28                  16089                  16443  \n",
       "29                  16443                  16943  \n",
       "30                  16943                  17443  \n",
       "31                  17443                  21893  \n",
       "32                  21893                  53857  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Categ_Folder</th>\n",
       "      <th>Dataset_File</th>\n",
       "      <th>min_global_record_idx</th>\n",
       "      <th>max_global_record_idx</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>animal_class</td>\n",
       "      <td>animal_class.csv</td>\n",
       "      <td>0</td>\n",
       "      <td>164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>animal_class</td>\n",
       "      <td>animal_class_conj.csv</td>\n",
       "      <td>164</td>\n",
       "      <td>664</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>animal_class</td>\n",
       "      <td>animal_class_disj.csv</td>\n",
       "      <td>664</td>\n",
       "      <td>1164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>animal_class</td>\n",
       "      <td>neg_animal_class.csv</td>\n",
       "      <td>1164</td>\n",
       "      <td>1328</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>cities</td>\n",
       "      <td>cities.csv</td>\n",
       "      <td>1328</td>\n",
       "      <td>2824</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>cities</td>\n",
       "      <td>cities_conj.csv</td>\n",
       "      <td>2824</td>\n",
       "      <td>4322</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>cities</td>\n",
       "      <td>cities_disj.csv</td>\n",
       "      <td>4322</td>\n",
       "      <td>4822</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>cities</td>\n",
       "      <td>neg_cities.csv</td>\n",
       "      <td>4822</td>\n",
       "      <td>6318</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>element_symb</td>\n",
       "      <td>element_symb.csv</td>\n",
       "      <td>6318</td>\n",
       "      <td>6504</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>element_symb</td>\n",
       "      <td>element_symb_conj.csv</td>\n",
       "      <td>6504</td>\n",
       "      <td>7004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>element_symb</td>\n",
       "      <td>element_symb_disj.csv</td>\n",
       "      <td>7004</td>\n",
       "      <td>7504</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>element_symb</td>\n",
       "      <td>neg_element_symb.csv</td>\n",
       "      <td>7504</td>\n",
       "      <td>7690</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>facts</td>\n",
       "      <td>facts.csv</td>\n",
       "      <td>7690</td>\n",
       "      <td>8251</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>facts</td>\n",
       "      <td>facts_conj.csv</td>\n",
       "      <td>8251</td>\n",
       "      <td>8751</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>facts</td>\n",
       "      <td>facts_disj.csv</td>\n",
       "      <td>8751</td>\n",
       "      <td>9251</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>facts</td>\n",
       "      <td>neg_facts.csv</td>\n",
       "      <td>9251</td>\n",
       "      <td>9812</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>inventors</td>\n",
       "      <td>inventors.csv</td>\n",
       "      <td>9812</td>\n",
       "      <td>10218</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>inventors</td>\n",
       "      <td>inventors_conj.csv</td>\n",
       "      <td>10218</td>\n",
       "      <td>10718</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>inventors</td>\n",
       "      <td>inventors_disj.csv</td>\n",
       "      <td>10718</td>\n",
       "      <td>11218</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>inventors</td>\n",
       "      <td>neg_inventors.csv</td>\n",
       "      <td>11218</td>\n",
       "      <td>11624</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>real_world_scenarios</td>\n",
       "      <td>ambiguous_lie.csv</td>\n",
       "      <td>11624</td>\n",
       "      <td>11642</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>real_world_scenarios</td>\n",
       "      <td>ambiguous_truthful_reply.csv</td>\n",
       "      <td>11642</td>\n",
       "      <td>11665</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>real_world_scenarios</td>\n",
       "      <td>honest_reply_despite_incentive_to_lie.csv</td>\n",
       "      <td>11665</td>\n",
       "      <td>11671</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>real_world_scenarios</td>\n",
       "      <td>unambiguous_lie.csv</td>\n",
       "      <td>11671</td>\n",
       "      <td>11720</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>real_world_scenarios</td>\n",
       "      <td>unambiguous_truthful_reply.csv</td>\n",
       "      <td>11720</td>\n",
       "      <td>11775</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>relative_comparison</td>\n",
       "      <td>larger_than.csv</td>\n",
       "      <td>11775</td>\n",
       "      <td>13755</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>relative_comparison</td>\n",
       "      <td>smaller_than.csv</td>\n",
       "      <td>13755</td>\n",
       "      <td>15735</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>sp_en_trans</td>\n",
       "      <td>neg_sp_en_trans.csv</td>\n",
       "      <td>15735</td>\n",
       "      <td>16089</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>sp_en_trans</td>\n",
       "      <td>sp_en_trans.csv</td>\n",
       "      <td>16089</td>\n",
       "      <td>16443</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>sp_en_trans</td>\n",
       "      <td>sp_en_trans_conj.csv</td>\n",
       "      <td>16443</td>\n",
       "      <td>16943</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>sp_en_trans</td>\n",
       "      <td>sp_en_trans_disj.csv</td>\n",
       "      <td>16943</td>\n",
       "      <td>17443</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>true_false</td>\n",
       "      <td>common_claim_true_false.csv</td>\n",
       "      <td>17443</td>\n",
       "      <td>21893</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>true_false</td>\n",
       "      <td>counterfact_true_false.csv</td>\n",
       "      <td>21893</td>\n",
       "      <td>53857</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 31
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-13T17:50:24.858458Z",
     "start_time": "2024-10-13T17:50:24.850718Z"
    }
   },
   "cell_type": "code",
   "source": "dset_folder_file_names_df.to_csv(dsets_folder / \"categ_dataset_pairs.csv\", index_label=\"Idx\")",
   "id": "74ad654017a2a31f",
   "outputs": [],
   "execution_count": 32
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-13T19:47:59.338296Z",
     "start_time": "2024-10-13T19:47:59.324416Z"
    }
   },
   "cell_type": "code",
   "source": [
    "dset_folder_file_names_df = pd.read_csv(dsets_folder / \"categ_dataset_pairs.csv\", index_col=\"Idx\")\n",
    "dset_folder_file_names_df"
   ],
   "id": "85775d4679e84c19",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "             Categ_Folder                               Dataset_File  \\\n",
       "Idx                                                                    \n",
       "0            animal_class                           animal_class.csv   \n",
       "1            animal_class                      animal_class_conj.csv   \n",
       "2            animal_class                      animal_class_disj.csv   \n",
       "3            animal_class                       neg_animal_class.csv   \n",
       "4                  cities                                 cities.csv   \n",
       "5                  cities                            cities_conj.csv   \n",
       "6                  cities                            cities_disj.csv   \n",
       "7                  cities                             neg_cities.csv   \n",
       "8            element_symb                           element_symb.csv   \n",
       "9            element_symb                      element_symb_conj.csv   \n",
       "10           element_symb                      element_symb_disj.csv   \n",
       "11           element_symb                       neg_element_symb.csv   \n",
       "12                  facts                                  facts.csv   \n",
       "13                  facts                             facts_conj.csv   \n",
       "14                  facts                             facts_disj.csv   \n",
       "15                  facts                              neg_facts.csv   \n",
       "16              inventors                              inventors.csv   \n",
       "17              inventors                         inventors_conj.csv   \n",
       "18              inventors                         inventors_disj.csv   \n",
       "19              inventors                          neg_inventors.csv   \n",
       "20   real_world_scenarios                          ambiguous_lie.csv   \n",
       "21   real_world_scenarios               ambiguous_truthful_reply.csv   \n",
       "22   real_world_scenarios  honest_reply_despite_incentive_to_lie.csv   \n",
       "23   real_world_scenarios                        unambiguous_lie.csv   \n",
       "24   real_world_scenarios             unambiguous_truthful_reply.csv   \n",
       "25    relative_comparison                            larger_than.csv   \n",
       "26    relative_comparison                           smaller_than.csv   \n",
       "27            sp_en_trans                        neg_sp_en_trans.csv   \n",
       "28            sp_en_trans                            sp_en_trans.csv   \n",
       "29            sp_en_trans                       sp_en_trans_conj.csv   \n",
       "30            sp_en_trans                       sp_en_trans_disj.csv   \n",
       "31             true_false                common_claim_true_false.csv   \n",
       "32             true_false                 counterfact_true_false.csv   \n",
       "\n",
       "     min_global_record_idx  max_global_record_idx  \n",
       "Idx                                                \n",
       "0                        0                    164  \n",
       "1                      164                    664  \n",
       "2                      664                   1164  \n",
       "3                     1164                   1328  \n",
       "4                     1328                   2824  \n",
       "5                     2824                   4322  \n",
       "6                     4322                   4822  \n",
       "7                     4822                   6318  \n",
       "8                     6318                   6504  \n",
       "9                     6504                   7004  \n",
       "10                    7004                   7504  \n",
       "11                    7504                   7690  \n",
       "12                    7690                   8251  \n",
       "13                    8251                   8751  \n",
       "14                    8751                   9251  \n",
       "15                    9251                   9812  \n",
       "16                    9812                  10218  \n",
       "17                   10218                  10718  \n",
       "18                   10718                  11218  \n",
       "19                   11218                  11624  \n",
       "20                   11624                  11642  \n",
       "21                   11642                  11665  \n",
       "22                   11665                  11671  \n",
       "23                   11671                  11720  \n",
       "24                   11720                  11775  \n",
       "25                   11775                  13755  \n",
       "26                   13755                  15735  \n",
       "27                   15735                  16089  \n",
       "28                   16089                  16443  \n",
       "29                   16443                  16943  \n",
       "30                   16943                  17443  \n",
       "31                   17443                  21893  \n",
       "32                   21893                  53857  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Categ_Folder</th>\n",
       "      <th>Dataset_File</th>\n",
       "      <th>min_global_record_idx</th>\n",
       "      <th>max_global_record_idx</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Idx</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>animal_class</td>\n",
       "      <td>animal_class.csv</td>\n",
       "      <td>0</td>\n",
       "      <td>164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>animal_class</td>\n",
       "      <td>animal_class_conj.csv</td>\n",
       "      <td>164</td>\n",
       "      <td>664</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>animal_class</td>\n",
       "      <td>animal_class_disj.csv</td>\n",
       "      <td>664</td>\n",
       "      <td>1164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>animal_class</td>\n",
       "      <td>neg_animal_class.csv</td>\n",
       "      <td>1164</td>\n",
       "      <td>1328</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>cities</td>\n",
       "      <td>cities.csv</td>\n",
       "      <td>1328</td>\n",
       "      <td>2824</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>cities</td>\n",
       "      <td>cities_conj.csv</td>\n",
       "      <td>2824</td>\n",
       "      <td>4322</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>cities</td>\n",
       "      <td>cities_disj.csv</td>\n",
       "      <td>4322</td>\n",
       "      <td>4822</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>cities</td>\n",
       "      <td>neg_cities.csv</td>\n",
       "      <td>4822</td>\n",
       "      <td>6318</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>element_symb</td>\n",
       "      <td>element_symb.csv</td>\n",
       "      <td>6318</td>\n",
       "      <td>6504</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>element_symb</td>\n",
       "      <td>element_symb_conj.csv</td>\n",
       "      <td>6504</td>\n",
       "      <td>7004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>element_symb</td>\n",
       "      <td>element_symb_disj.csv</td>\n",
       "      <td>7004</td>\n",
       "      <td>7504</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>element_symb</td>\n",
       "      <td>neg_element_symb.csv</td>\n",
       "      <td>7504</td>\n",
       "      <td>7690</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>facts</td>\n",
       "      <td>facts.csv</td>\n",
       "      <td>7690</td>\n",
       "      <td>8251</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>facts</td>\n",
       "      <td>facts_conj.csv</td>\n",
       "      <td>8251</td>\n",
       "      <td>8751</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>facts</td>\n",
       "      <td>facts_disj.csv</td>\n",
       "      <td>8751</td>\n",
       "      <td>9251</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>facts</td>\n",
       "      <td>neg_facts.csv</td>\n",
       "      <td>9251</td>\n",
       "      <td>9812</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>inventors</td>\n",
       "      <td>inventors.csv</td>\n",
       "      <td>9812</td>\n",
       "      <td>10218</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>inventors</td>\n",
       "      <td>inventors_conj.csv</td>\n",
       "      <td>10218</td>\n",
       "      <td>10718</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>inventors</td>\n",
       "      <td>inventors_disj.csv</td>\n",
       "      <td>10718</td>\n",
       "      <td>11218</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>inventors</td>\n",
       "      <td>neg_inventors.csv</td>\n",
       "      <td>11218</td>\n",
       "      <td>11624</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>real_world_scenarios</td>\n",
       "      <td>ambiguous_lie.csv</td>\n",
       "      <td>11624</td>\n",
       "      <td>11642</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>real_world_scenarios</td>\n",
       "      <td>ambiguous_truthful_reply.csv</td>\n",
       "      <td>11642</td>\n",
       "      <td>11665</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>real_world_scenarios</td>\n",
       "      <td>honest_reply_despite_incentive_to_lie.csv</td>\n",
       "      <td>11665</td>\n",
       "      <td>11671</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>real_world_scenarios</td>\n",
       "      <td>unambiguous_lie.csv</td>\n",
       "      <td>11671</td>\n",
       "      <td>11720</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>real_world_scenarios</td>\n",
       "      <td>unambiguous_truthful_reply.csv</td>\n",
       "      <td>11720</td>\n",
       "      <td>11775</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>relative_comparison</td>\n",
       "      <td>larger_than.csv</td>\n",
       "      <td>11775</td>\n",
       "      <td>13755</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>relative_comparison</td>\n",
       "      <td>smaller_than.csv</td>\n",
       "      <td>13755</td>\n",
       "      <td>15735</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>sp_en_trans</td>\n",
       "      <td>neg_sp_en_trans.csv</td>\n",
       "      <td>15735</td>\n",
       "      <td>16089</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>sp_en_trans</td>\n",
       "      <td>sp_en_trans.csv</td>\n",
       "      <td>16089</td>\n",
       "      <td>16443</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>sp_en_trans</td>\n",
       "      <td>sp_en_trans_conj.csv</td>\n",
       "      <td>16443</td>\n",
       "      <td>16943</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>sp_en_trans</td>\n",
       "      <td>sp_en_trans_disj.csv</td>\n",
       "      <td>16943</td>\n",
       "      <td>17443</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>true_false</td>\n",
       "      <td>common_claim_true_false.csv</td>\n",
       "      <td>17443</td>\n",
       "      <td>21893</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>true_false</td>\n",
       "      <td>counterfact_true_false.csv</td>\n",
       "      <td>21893</td>\n",
       "      <td>53857</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 41
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-13T21:21:03.436753Z",
     "start_time": "2024-10-13T21:21:00.932304Z"
    }
   },
   "cell_type": "code",
   "source": [
    "for (idx, categ_folder_nm, dset_file_nm, _, _) in dset_folder_file_names_df.itertuples():\n",
    "    curr_dset_df = pd.read_csv(dsets_folder / categ_folder_nm / dset_file_nm)\n",
    "    if 'statement' not in curr_dset_df.columns or 'label' not in curr_dset_df.columns:\n",
    "        print(f\"WARNING- in category {categ_folder_nm}, dataset {dset_file_nm} was read with column names that're inconsistent with the other datasets: {curr_dset_df.columns}\")\n",
    "    else:\n",
    "        for index, row in curr_dset_df.iterrows():\n",
    "            if row['statement'][-1] not in [\".\", \"!\", \"?\"]:\n",
    "                print(f\"WARNING- in category {categ_folder_nm}, dataset {dset_file_nm}'s {index}th row has value with no period at the end: {row['statement']}\")"
   ],
   "id": "7c6ab51d59b4af44",
   "outputs": [],
   "execution_count": 44
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-13T21:58:10.201955Z",
     "start_time": "2024-10-13T21:58:10.177890Z"
    }
   },
   "cell_type": "code",
   "source": [
    "dset_folder_file_names_df[\"is_negated\"] = False\n",
    "dset_folder_file_names_df[\"is_conj\"] = False\n",
    "dset_folder_file_names_df[\"is_disj\"] = False\n",
    "#\"is_other\" refers to the \"other\" types of categories, the ones that don't have the simple \"positive-negative-conjunction-disjunction\" structure of dataset variants \n",
    "dset_folder_file_names_df[\"is_other\"] = False"
   ],
   "id": "54e686c365989232",
   "outputs": [],
   "execution_count": 49
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-13T21:58:10.786369Z",
     "start_time": "2024-10-13T21:58:10.772275Z"
    }
   },
   "cell_type": "code",
   "source": [
    "for (idx, categ_folder_nm, dset_file_nm, _,_,_,_,_,_) in dset_folder_file_names_df.itertuples():\n",
    "    if categ_folder_nm in [\"real_world_scenarios\", \"relative_comparison\", \"true_false\"]:\n",
    "        dset_folder_file_names_df.at[idx, 'is_other'] = True\n",
    "    elif dset_file_nm.startswith(\"neg_\"):\n",
    "        dset_folder_file_names_df.at[idx, 'is_negated'] = True\n",
    "    elif dset_file_nm.endswith(\"_conj.csv\"):\n",
    "        dset_folder_file_names_df.at[idx, 'is_conj'] = True\n",
    "    elif dset_file_nm.endswith(\"_disj.csv\"):\n",
    "        dset_folder_file_names_df.at[idx, 'is_disj'] = True\n"
   ],
   "id": "322fc182392dc366",
   "outputs": [],
   "execution_count": 50
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-13T21:58:11.602330Z",
     "start_time": "2024-10-13T21:58:11.579211Z"
    }
   },
   "cell_type": "code",
   "source": "dset_folder_file_names_df",
   "id": "ebec765e56a95e08",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "             Categ_Folder                               Dataset_File  \\\n",
       "Idx                                                                    \n",
       "0            animal_class                           animal_class.csv   \n",
       "1            animal_class                      animal_class_conj.csv   \n",
       "2            animal_class                      animal_class_disj.csv   \n",
       "3            animal_class                       neg_animal_class.csv   \n",
       "4                  cities                                 cities.csv   \n",
       "5                  cities                            cities_conj.csv   \n",
       "6                  cities                            cities_disj.csv   \n",
       "7                  cities                             neg_cities.csv   \n",
       "8            element_symb                           element_symb.csv   \n",
       "9            element_symb                      element_symb_conj.csv   \n",
       "10           element_symb                      element_symb_disj.csv   \n",
       "11           element_symb                       neg_element_symb.csv   \n",
       "12                  facts                                  facts.csv   \n",
       "13                  facts                             facts_conj.csv   \n",
       "14                  facts                             facts_disj.csv   \n",
       "15                  facts                              neg_facts.csv   \n",
       "16              inventors                              inventors.csv   \n",
       "17              inventors                         inventors_conj.csv   \n",
       "18              inventors                         inventors_disj.csv   \n",
       "19              inventors                          neg_inventors.csv   \n",
       "20   real_world_scenarios                          ambiguous_lie.csv   \n",
       "21   real_world_scenarios               ambiguous_truthful_reply.csv   \n",
       "22   real_world_scenarios  honest_reply_despite_incentive_to_lie.csv   \n",
       "23   real_world_scenarios                        unambiguous_lie.csv   \n",
       "24   real_world_scenarios             unambiguous_truthful_reply.csv   \n",
       "25    relative_comparison                            larger_than.csv   \n",
       "26    relative_comparison                           smaller_than.csv   \n",
       "27            sp_en_trans                        neg_sp_en_trans.csv   \n",
       "28            sp_en_trans                            sp_en_trans.csv   \n",
       "29            sp_en_trans                       sp_en_trans_conj.csv   \n",
       "30            sp_en_trans                       sp_en_trans_disj.csv   \n",
       "31             true_false                common_claim_true_false.csv   \n",
       "32             true_false                 counterfact_true_false.csv   \n",
       "\n",
       "     min_global_record_idx  max_global_record_idx  is_negated  is_conj  \\\n",
       "Idx                                                                      \n",
       "0                        0                    164       False    False   \n",
       "1                      164                    664       False     True   \n",
       "2                      664                   1164       False    False   \n",
       "3                     1164                   1328        True    False   \n",
       "4                     1328                   2824       False    False   \n",
       "5                     2824                   4322       False     True   \n",
       "6                     4322                   4822       False    False   \n",
       "7                     4822                   6318        True    False   \n",
       "8                     6318                   6504       False    False   \n",
       "9                     6504                   7004       False     True   \n",
       "10                    7004                   7504       False    False   \n",
       "11                    7504                   7690        True    False   \n",
       "12                    7690                   8251       False    False   \n",
       "13                    8251                   8751       False     True   \n",
       "14                    8751                   9251       False    False   \n",
       "15                    9251                   9812        True    False   \n",
       "16                    9812                  10218       False    False   \n",
       "17                   10218                  10718       False     True   \n",
       "18                   10718                  11218       False    False   \n",
       "19                   11218                  11624        True    False   \n",
       "20                   11624                  11642       False    False   \n",
       "21                   11642                  11665       False    False   \n",
       "22                   11665                  11671       False    False   \n",
       "23                   11671                  11720       False    False   \n",
       "24                   11720                  11775       False    False   \n",
       "25                   11775                  13755       False    False   \n",
       "26                   13755                  15735       False    False   \n",
       "27                   15735                  16089        True    False   \n",
       "28                   16089                  16443       False    False   \n",
       "29                   16443                  16943       False     True   \n",
       "30                   16943                  17443       False    False   \n",
       "31                   17443                  21893       False    False   \n",
       "32                   21893                  53857       False    False   \n",
       "\n",
       "     is_disj  is_other  \n",
       "Idx                     \n",
       "0      False     False  \n",
       "1      False     False  \n",
       "2       True     False  \n",
       "3      False     False  \n",
       "4      False     False  \n",
       "5      False     False  \n",
       "6       True     False  \n",
       "7      False     False  \n",
       "8      False     False  \n",
       "9      False     False  \n",
       "10      True     False  \n",
       "11     False     False  \n",
       "12     False     False  \n",
       "13     False     False  \n",
       "14      True     False  \n",
       "15     False     False  \n",
       "16     False     False  \n",
       "17     False     False  \n",
       "18      True     False  \n",
       "19     False     False  \n",
       "20     False      True  \n",
       "21     False      True  \n",
       "22     False      True  \n",
       "23     False      True  \n",
       "24     False      True  \n",
       "25     False      True  \n",
       "26     False      True  \n",
       "27     False     False  \n",
       "28     False     False  \n",
       "29     False     False  \n",
       "30      True     False  \n",
       "31     False      True  \n",
       "32     False      True  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Categ_Folder</th>\n",
       "      <th>Dataset_File</th>\n",
       "      <th>min_global_record_idx</th>\n",
       "      <th>max_global_record_idx</th>\n",
       "      <th>is_negated</th>\n",
       "      <th>is_conj</th>\n",
       "      <th>is_disj</th>\n",
       "      <th>is_other</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Idx</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>animal_class</td>\n",
       "      <td>animal_class.csv</td>\n",
       "      <td>0</td>\n",
       "      <td>164</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>animal_class</td>\n",
       "      <td>animal_class_conj.csv</td>\n",
       "      <td>164</td>\n",
       "      <td>664</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>animal_class</td>\n",
       "      <td>animal_class_disj.csv</td>\n",
       "      <td>664</td>\n",
       "      <td>1164</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>animal_class</td>\n",
       "      <td>neg_animal_class.csv</td>\n",
       "      <td>1164</td>\n",
       "      <td>1328</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>cities</td>\n",
       "      <td>cities.csv</td>\n",
       "      <td>1328</td>\n",
       "      <td>2824</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>cities</td>\n",
       "      <td>cities_conj.csv</td>\n",
       "      <td>2824</td>\n",
       "      <td>4322</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>cities</td>\n",
       "      <td>cities_disj.csv</td>\n",
       "      <td>4322</td>\n",
       "      <td>4822</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>cities</td>\n",
       "      <td>neg_cities.csv</td>\n",
       "      <td>4822</td>\n",
       "      <td>6318</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>element_symb</td>\n",
       "      <td>element_symb.csv</td>\n",
       "      <td>6318</td>\n",
       "      <td>6504</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>element_symb</td>\n",
       "      <td>element_symb_conj.csv</td>\n",
       "      <td>6504</td>\n",
       "      <td>7004</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>element_symb</td>\n",
       "      <td>element_symb_disj.csv</td>\n",
       "      <td>7004</td>\n",
       "      <td>7504</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>element_symb</td>\n",
       "      <td>neg_element_symb.csv</td>\n",
       "      <td>7504</td>\n",
       "      <td>7690</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>facts</td>\n",
       "      <td>facts.csv</td>\n",
       "      <td>7690</td>\n",
       "      <td>8251</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>facts</td>\n",
       "      <td>facts_conj.csv</td>\n",
       "      <td>8251</td>\n",
       "      <td>8751</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>facts</td>\n",
       "      <td>facts_disj.csv</td>\n",
       "      <td>8751</td>\n",
       "      <td>9251</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>facts</td>\n",
       "      <td>neg_facts.csv</td>\n",
       "      <td>9251</td>\n",
       "      <td>9812</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>inventors</td>\n",
       "      <td>inventors.csv</td>\n",
       "      <td>9812</td>\n",
       "      <td>10218</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>inventors</td>\n",
       "      <td>inventors_conj.csv</td>\n",
       "      <td>10218</td>\n",
       "      <td>10718</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>inventors</td>\n",
       "      <td>inventors_disj.csv</td>\n",
       "      <td>10718</td>\n",
       "      <td>11218</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>inventors</td>\n",
       "      <td>neg_inventors.csv</td>\n",
       "      <td>11218</td>\n",
       "      <td>11624</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>real_world_scenarios</td>\n",
       "      <td>ambiguous_lie.csv</td>\n",
       "      <td>11624</td>\n",
       "      <td>11642</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>real_world_scenarios</td>\n",
       "      <td>ambiguous_truthful_reply.csv</td>\n",
       "      <td>11642</td>\n",
       "      <td>11665</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>real_world_scenarios</td>\n",
       "      <td>honest_reply_despite_incentive_to_lie.csv</td>\n",
       "      <td>11665</td>\n",
       "      <td>11671</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>real_world_scenarios</td>\n",
       "      <td>unambiguous_lie.csv</td>\n",
       "      <td>11671</td>\n",
       "      <td>11720</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>real_world_scenarios</td>\n",
       "      <td>unambiguous_truthful_reply.csv</td>\n",
       "      <td>11720</td>\n",
       "      <td>11775</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>relative_comparison</td>\n",
       "      <td>larger_than.csv</td>\n",
       "      <td>11775</td>\n",
       "      <td>13755</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>relative_comparison</td>\n",
       "      <td>smaller_than.csv</td>\n",
       "      <td>13755</td>\n",
       "      <td>15735</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>sp_en_trans</td>\n",
       "      <td>neg_sp_en_trans.csv</td>\n",
       "      <td>15735</td>\n",
       "      <td>16089</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>sp_en_trans</td>\n",
       "      <td>sp_en_trans.csv</td>\n",
       "      <td>16089</td>\n",
       "      <td>16443</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>sp_en_trans</td>\n",
       "      <td>sp_en_trans_conj.csv</td>\n",
       "      <td>16443</td>\n",
       "      <td>16943</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>sp_en_trans</td>\n",
       "      <td>sp_en_trans_disj.csv</td>\n",
       "      <td>16943</td>\n",
       "      <td>17443</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>true_false</td>\n",
       "      <td>common_claim_true_false.csv</td>\n",
       "      <td>17443</td>\n",
       "      <td>21893</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>true_false</td>\n",
       "      <td>counterfact_true_false.csv</td>\n",
       "      <td>21893</td>\n",
       "      <td>53857</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 51
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-13T21:59:11.157745Z",
     "start_time": "2024-10-13T21:59:11.142572Z"
    }
   },
   "cell_type": "code",
   "source": "dset_folder_file_names_df.to_csv(dsets_index_path, index_label=\"Idx\")",
   "id": "9ddbaff674f8ecf2",
   "outputs": [],
   "execution_count": 52
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}