create_tables.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create SMC claim dataset tables\n",
    "This notebook guides through the creation of tables from the SQLite database. Flat data tables are needed for downstream tasks, e.g., annotation and model training. \n",
    "\n",
    "**Table of content:**\n",
    "1. Create metadata table\n",
    "2. Create data table\n",
    "3. Create slices to be labeled\n",
    "4. Create SMC_2000 dataset\n",
    "5. Create SMC_1000 dataset\n",
    "6. Create SMC_Full dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sqlite3\n",
    "import os\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "from config import DB_PATH, BASE_DIR"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Create metadata table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "connection = sqlite3.connect(DB_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pb_ID</th>\n",
       "      <th>title</th>\n",
       "      <th>date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Welche nationale Teststrategie bringt uns opti...</td>\n",
       "      <td>09.10.2020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Früherkennung von Alzheimer – Je früher, desto...</td>\n",
       "      <td>15.09.2017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Desinformation vor der Europawahl – Wie groß i...</td>\n",
       "      <td>12.04.2019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>Wie lassen sich der Kampf gegen den Klimawande...</td>\n",
       "      <td>08.06.2021</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>Zukünftige Teststrategien auf SARS-CoV-2 und  ...</td>\n",
       "      <td>09.04.2020</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   pb_ID                                              title        date\n",
       "0      1  Welche nationale Teststrategie bringt uns opti...  09.10.2020\n",
       "1      2  Früherkennung von Alzheimer – Je früher, desto...  15.09.2017\n",
       "2      3  Desinformation vor der Europawahl – Wie groß i...  12.04.2019\n",
       "3      4  Wie lassen sich der Kampf gegen den Klimawande...  08.06.2021\n",
       "4      5  Zukünftige Teststrategien auf SARS-CoV-2 und  ...  09.04.2020"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metadata_table = pd.read_sql_query(\"SELECT pb_ID, title, date FROM Press_Briefing\", connection)  # + topic , resort\n",
    "metadata_table.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Create data table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence_ID</th>\n",
       "      <th>title</th>\n",
       "      <th>sentence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Welche nationale Teststrategie bringt uns opti...</td>\n",
       "      <td>Transkript</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Welche nationale Teststrategie bringt uns opti...</td>\n",
       "      <td>Herzlich willkommen zum heutigen Press Briefin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Welche nationale Teststrategie bringt uns opti...</td>\n",
       "      <td>Mein Name ist Moderatorin, ich bin Redakteurin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>Welche nationale Teststrategie bringt uns opti...</td>\n",
       "      <td>Ich begrüße die teilnehmenden Journalisten und...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>Welche nationale Teststrategie bringt uns opti...</td>\n",
       "      <td>Der Herbst ist da, die Infektionszahlen steige...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   sentence_ID                                              title  \\\n",
       "0            1  Welche nationale Teststrategie bringt uns opti...   \n",
       "1            2  Welche nationale Teststrategie bringt uns opti...   \n",
       "2            3  Welche nationale Teststrategie bringt uns opti...   \n",
       "3            4  Welche nationale Teststrategie bringt uns opti...   \n",
       "4            5  Welche nationale Teststrategie bringt uns opti...   \n",
       "\n",
       "                                            sentence  \n",
       "0                                         Transkript  \n",
       "1  Herzlich willkommen zum heutigen Press Briefin...  \n",
       "2  Mein Name ist Moderatorin, ich bin Redakteurin...  \n",
       "3  Ich begrüße die teilnehmenden Journalisten und...  \n",
       "4  Der Herbst ist da, die Infektionszahlen steige...  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_table = pd.read_sql_query(\"\"\"SELECT Sentence.sentence_ID, title, sentence\n",
    "        FROM Sentence \n",
    "        JOIN Press_Briefing ON Sentence.pb_ID = Press_Briefing.pb_ID\"\"\", connection)\n",
    "data_table.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_table.to_csv(os.path.join(BASE_DIR, \"SMC_claim_sentences.csv\"), index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Create slices to be labeled\n",
    "Sentences with over 99% claim probability:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence_ID</th>\n",
       "      <th>title</th>\n",
       "      <th>sentence</th>\n",
       "      <th>prob</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Welche nationale Teststrategie bringt uns opti...</td>\n",
       "      <td>Transkript</td>\n",
       "      <td>0.001865</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Welche nationale Teststrategie bringt uns opti...</td>\n",
       "      <td>Herzlich willkommen zum heutigen Press Briefin...</td>\n",
       "      <td>0.002239</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Welche nationale Teststrategie bringt uns opti...</td>\n",
       "      <td>Mein Name ist Moderatorin, ich bin Redakteurin...</td>\n",
       "      <td>0.002861</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>Welche nationale Teststrategie bringt uns opti...</td>\n",
       "      <td>Ich begrüße die teilnehmenden Journalisten und...</td>\n",
       "      <td>0.004546</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>Welche nationale Teststrategie bringt uns opti...</td>\n",
       "      <td>Der Herbst ist da, die Infektionszahlen steige...</td>\n",
       "      <td>0.003052</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   sentence_ID                                              title  \\\n",
       "0            1  Welche nationale Teststrategie bringt uns opti...   \n",
       "1            2  Welche nationale Teststrategie bringt uns opti...   \n",
       "2            3  Welche nationale Teststrategie bringt uns opti...   \n",
       "3            4  Welche nationale Teststrategie bringt uns opti...   \n",
       "4            5  Welche nationale Teststrategie bringt uns opti...   \n",
       "\n",
       "                                            sentence      prob label  \n",
       "0                                         Transkript  0.001865  None  \n",
       "1  Herzlich willkommen zum heutigen Press Briefin...  0.002239  None  \n",
       "2  Mein Name ist Moderatorin, ich bin Redakteurin...  0.002861  None  \n",
       "3  Ich begrüße die teilnehmenden Journalisten und...  0.004546  None  \n",
       "4  Der Herbst ist da, die Infektionszahlen steige...  0.003052  None  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv(os.path.join(\"data\", \"SMC_dataset\", \"pre_labeled\", \"SMC_claim_setences_pre.csv\"), names=[\"sentence_ID\", \"title\", \"sentence\", \"prob\"])\n",
    "data[\"label\"] = None  # Column for the label\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence_ID</th>\n",
       "      <th>title</th>\n",
       "      <th>sentence</th>\n",
       "      <th>prob</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>20</td>\n",
       "      <td>Welche nationale Teststrategie bringt uns opti...</td>\n",
       "      <td>Bitte wählen Sie die Option an alle Teilnehmer...</td>\n",
       "      <td>0.992950</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>26</td>\n",
       "      <td>Welche nationale Teststrategie bringt uns opti...</td>\n",
       "      <td>Aber ich möchte nochmal betonen, dass aus mein...</td>\n",
       "      <td>0.992971</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>31</td>\n",
       "      <td>Welche nationale Teststrategie bringt uns opti...</td>\n",
       "      <td>Wenn dieser Test positiv ausfällt, ist aber au...</td>\n",
       "      <td>0.993059</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>35</td>\n",
       "      <td>Welche nationale Teststrategie bringt uns opti...</td>\n",
       "      <td>Genau das zweite, die Sensitivität ist nicht s...</td>\n",
       "      <td>0.992652</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>43</td>\n",
       "      <td>Welche nationale Teststrategie bringt uns opti...</td>\n",
       "      <td>Diejenigen werden aber unter Umständen auch ni...</td>\n",
       "      <td>0.993070</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    sentence_ID                                              title  \\\n",
       "19           20  Welche nationale Teststrategie bringt uns opti...   \n",
       "25           26  Welche nationale Teststrategie bringt uns opti...   \n",
       "30           31  Welche nationale Teststrategie bringt uns opti...   \n",
       "34           35  Welche nationale Teststrategie bringt uns opti...   \n",
       "42           43  Welche nationale Teststrategie bringt uns opti...   \n",
       "\n",
       "                                             sentence      prob label  \n",
       "19  Bitte wählen Sie die Option an alle Teilnehmer...  0.992950  None  \n",
       "25  Aber ich möchte nochmal betonen, dass aus mein...  0.992971  None  \n",
       "30  Wenn dieser Test positiv ausfällt, ist aber au...  0.993059  None  \n",
       "34  Genau das zweite, die Sensitivität ist nicht s...  0.992652  None  \n",
       "42  Diejenigen werden aber unter Umständen auch ni...  0.993070  None  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_99 = data[data[\"prob\"]>= 0.99]\n",
    "dataset_99.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_99.to_json(os.path.join(\"data\", \"SMC_dataset\", \"pre_labeled\", \"SMC_claim_sentences_claims_99.jsonl\"), orient='records', lines=True)  # aka v2.jsonl"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sentences with claim probability between 20% and 80%:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_2706/1292459789.py:1: FutureWarning: Boolean inputs to the `inclusive` argument are deprecated infavour of `both` or `neither`.\n",
      "  dataset_20_80 = data[data[\"prob\"].between(0.20, 0.80, inclusive=True)]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence_ID</th>\n",
       "      <th>title</th>\n",
       "      <th>sentence</th>\n",
       "      <th>prob</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>15</td>\n",
       "      <td>Welche nationale Teststrategie bringt uns opti...</td>\n",
       "      <td>Allerdings diskutieren auch weitere Bereiche z...</td>\n",
       "      <td>0.353402</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>22</td>\n",
       "      <td>Welche nationale Teststrategie bringt uns opti...</td>\n",
       "      <td>Sie ist Direktorin des Instituts für medizinis...</td>\n",
       "      <td>0.366258</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>23</td>\n",
       "      <td>Welche nationale Teststrategie bringt uns opti...</td>\n",
       "      <td>Hallo, Frau Ciesek.</td>\n",
       "      <td>0.285915</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>36</td>\n",
       "      <td>Welche nationale Teststrategie bringt uns opti...</td>\n",
       "      <td>Die Viruslast ist am höchsten kurz vor Auftrit...</td>\n",
       "      <td>0.262825</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>45</td>\n",
       "      <td>Welche nationale Teststrategie bringt uns opti...</td>\n",
       "      <td>Natürlich würde die PCR es ein bisschen früher...</td>\n",
       "      <td>0.238853</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    sentence_ID                                              title  \\\n",
       "14           15  Welche nationale Teststrategie bringt uns opti...   \n",
       "21           22  Welche nationale Teststrategie bringt uns opti...   \n",
       "22           23  Welche nationale Teststrategie bringt uns opti...   \n",
       "35           36  Welche nationale Teststrategie bringt uns opti...   \n",
       "44           45  Welche nationale Teststrategie bringt uns opti...   \n",
       "\n",
       "                                             sentence      prob label  \n",
       "14  Allerdings diskutieren auch weitere Bereiche z...  0.353402  None  \n",
       "21  Sie ist Direktorin des Instituts für medizinis...  0.366258  None  \n",
       "22                                Hallo, Frau Ciesek.  0.285915  None  \n",
       "35  Die Viruslast ist am höchsten kurz vor Auftrit...  0.262825  None  \n",
       "44  Natürlich würde die PCR es ein bisschen früher...  0.238853  None  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_20_80 = data[data[\"prob\"].between(0.20, 0.80, inclusive=True)]\n",
    "dataset_20_80.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_20_80.to_json(os.path.join(\"data\", \"SMC_dataset\", \"pre_labeled\", \"SMC_claim_sentences_claims_20_80.jsonl\"), orient='records', lines=True)  # aka v2.jsonl"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. SMC_2000 dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_top99 = pd.read_json(os.path.join(BASE_DIR, \"labeled\", \"SMC_claim_sentences_claims_99_labeled.jsonl\"), lines=True)\n",
    "dataset_top99 = dataset_top99.rename(columns={\"data\":\"Sentence\", \"label\": \"Claim\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "complete_claims = dataset_top99.explode(\"Claim\")[dataset_top99.explode(\"Claim\")[\"Claim\"].isin([\"complete_claim\", \"No_claim\"])]\n",
    "complete_claims[\"Claim\"] = complete_claims[\"Claim\"].apply(lambda x: False if \"No_claim\" in x else True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>Sentence</th>\n",
       "      <th>prob</th>\n",
       "      <th>title</th>\n",
       "      <th>sentence_ID</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Claim</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>False</th>\n",
       "      <td>351</td>\n",
       "      <td>351</td>\n",
       "      <td>351</td>\n",
       "      <td>351</td>\n",
       "      <td>351</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True</th>\n",
       "      <td>411</td>\n",
       "      <td>411</td>\n",
       "      <td>411</td>\n",
       "      <td>411</td>\n",
       "      <td>411</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        id  Sentence  prob  title  sentence_ID\n",
       "Claim                                         \n",
       "False  351       351   351    351          351\n",
       "True   411       411   411    411          411"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "complete_claims.groupby(by=\"Claim\").count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_2018 = pd.read_csv(os.path.join(BASE_DIR, \"claim_sentence_search_de_g.csv\"))  # sample from translated dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "false_samples = dataset_2018[dataset_2018[\"Claim\"]==False].sample(411-351)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "SMC_2000 = pd.concat([false_samples, complete_claims])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "SMC_2000 = SMC_2000[[\"Sentence\", \"Claim\", \"Article\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "SMC_2000.to_csv(os.path.join(BASE_DIR, \"SMC_2000.csv\"), index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5. SMC_1000 dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_top99_3k = pd.read_json(os.path.join(BASE_DIR, \"labeled\", \"SMC_claim_sentences_claims_99_labeled_3k.jsonl\"), lines=True)\n",
    "dataset_top99_3k = dataset_top99_3k.rename(columns={\"data\":\"Sentence\", \"label\": \"Claim\", \"title\":\"Article\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "complete_claim = dataset_top99_3k.explode(\"Claim\")[dataset_top99_3k.explode(\"Claim\")[\"Claim\"].isin([\"complete_claim\", \"No_claim\"])]\n",
    "complete_claim[\"Claim\"] = complete_claim[\"Claim\"].apply(lambda x: False if \"No_claim\" in x else True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>Sentence</th>\n",
       "      <th>prob</th>\n",
       "      <th>Article</th>\n",
       "      <th>sentence_ID</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Claim</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>False</th>\n",
       "      <td>557</td>\n",
       "      <td>557</td>\n",
       "      <td>557</td>\n",
       "      <td>557</td>\n",
       "      <td>557</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True</th>\n",
       "      <td>588</td>\n",
       "      <td>588</td>\n",
       "      <td>588</td>\n",
       "      <td>588</td>\n",
       "      <td>588</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        id  Sentence  prob  Article  sentence_ID\n",
       "Claim                                           \n",
       "False  557       557   557      557          557\n",
       "True   588       588   588      588          588"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "complete_claim.groupby(by=\"Claim\").count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "false_samples = dataset_2018[dataset_2018[\"Claim\"]==False].sample(588-557)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SMC_1000 = pd.concat([false_samples, complete_claim])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SMC_1000 = SMC_1000[[\"Sentence\", \"Claim\", \"Article\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SMC_1000.to_csv(os.path.join(BASE_DIR, \"SMC_1000.csv\"), index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6. SMC_Full dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SMC_Full = pd.concat([SMC_2000, SMC_1000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SMC_Full.to_csv(os.path.join(BASE_DIR, \"SMC_Full.csv\"), index=False)"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "949777d72b0d2535278d3dc13498b2535136f6dfe0678499012e853ee9abcab1"
  },
  "kernelspec": {
   "display_name": "Python 3.9.7 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}