PrepareDataFiles.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import itertools\n",
    "from pprint import pprint\n",
    "import pickle\n",
    "import pandas as pd\n",
    "import requests\n",
    "from tqdm import tqdm\n",
    "from sklearn.model_selection import train_test_split\n",
    "from scipy import stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv(\"\") # Path to metadata file provided by NEwsela"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(data['language'].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_stats():\n",
    "    # Distribution of y and y_lexile\n",
    "    text_counts = data['grade_level'].value_counts()\n",
    "\n",
    "    print('Categorial grade_level')\n",
    "    print(pd.DataFrame({\n",
    "        '#text': text_counts,\n",
    "        '%text': (100 * text_counts / text_counts.sum()).round(2),\n",
    "    }).sort_index())\n",
    "\n",
    "    dfs = [data]\n",
    "\n",
    "    stats_d = {'name': ['text'],\n",
    "             '#': [len(df['grade_level']) for df in dfs],\n",
    "             'min': [df['grade_level'].min() for df in dfs],\n",
    "             'max': [df['grade_level'].max() for df in dfs],\n",
    "             'mean': [df['grade_level'].mean() for df in dfs],\n",
    "             'std': [df['grade_level'].std() for df in dfs]\n",
    "            }\n",
    "\n",
    "    stats_df = pd.DataFrame(stats_d)\n",
    "    stats_df.set_index('name')\n",
    "\n",
    "    print('Continuous grade_level')\n",
    "    print(stats_df)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_stats()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Grade Stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grade_groups = data.groupby(['grade_level','language'])['filename'].apply(list).reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grades_lang = {}\n",
    "for index, row in grade_groups.iterrows():\n",
    "    y = row['grade_level']\n",
    "    ids = row['filename']\n",
    "    lang = row['language']\n",
    "    if y in grades_lang:\n",
    "        grades_lang[y][lang] = ids\n",
    "    else:\n",
    "        grades_lang[y] = {}\n",
    "        grades_lang[y][lang] = ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import collections\n",
    "import functools\n",
    "import os\n",
    "\n",
    "path = \"\" # Path to articles\n",
    "\n",
    "def iterate_data(data):\n",
    "    if isinstance(data, str):\n",
    "        assert os.path.exists(data), f\"path `{data}` does not exist!\"\n",
    "        with open(data, \"r\") as f:\n",
    "            for line in f:\n",
    "                if len(line.strip()) > 0:\n",
    "                    yield line\n",
    "\n",
    "    elif isinstance(data, collections.Iterable):\n",
    "        for x in data:\n",
    "            yield x\n",
    "\n",
    "def get_stats_data(ids):\n",
    "    _vocab = collections.Counter()\n",
    "    _data = []\n",
    "    for filename in ids:\n",
    "        for line in iterate_data(path + filename):\n",
    "            tokens = line.lower().split()\n",
    "            _vocab.update(tokens)\n",
    "            _data.append(tokens)\n",
    "    return _vocab, _data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lang=\"en\"\n",
    "for i in range(2, 13):\n",
    "    _vocab, _data = get_stats_data(grades_lang[i][lang])\n",
    "    length_stats = [len(x) for x in _data]\n",
    "    print(\"Grade: \" + str(i))\n",
    "    print(\"Number of Articles: \" + str(len(grades_lang[i][lang])))\n",
    "    print(\"Average length of the sentence:\"  + str(sum(length_stats)/len(length_stats)))\n",
    "    print(\"Number of unique tokens: \" + str(len(_vocab)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Creating data groups by language"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_slug(x):\n",
    "    x = x.replace(\"-spanish\", \"\")\n",
    "    return x\n",
    "\n",
    "data[\"slug\"] = data[\"slug\"].apply(clean_slug)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "news_groups = data.groupby(['slug','language'])['filename'].apply(list).reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "news_groups.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Translating spanish articles to english"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_lang_ids(flang):\n",
    "    lang_ids = set()\n",
    "    for index, row in news_groups.iterrows():\n",
    "        if row['language'] == flang:\n",
    "            lang_ids |= set(row['filename'])\n",
    "    return lang_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "from time import sleep\n",
    "from google_api_translate import Translator, TextUtils\n",
    "\n",
    "creds_path = \"\" # Google Translate Credentials json file path\n",
    "\n",
    "\n",
    "def create_translation(src_file, output_file):\n",
    "    lines = open(src_file, encoding='utf-8').read().strip().split('\\n')\n",
    "    if not os.path.exists(output_file):\n",
    "        with open(output_file, \"w\") as f:\n",
    "            for line in lines:\n",
    "                if(line!=\"\"):\n",
    "                    success = False\n",
    "                    while(not success):\n",
    "                        try:\n",
    "                            trans = Translator(creds_path=creds_path).translate(text=line, target_language='en')\n",
    "                            f.write(trans.text if trans.text!=\"\" else \"\\n\\n\")\n",
    "                            success = True\n",
    "                        except:\n",
    "                            sleep(100)\n",
    "                            continue\n",
    "                else:\n",
    "                    f.write(\"\\n\\n\")             \n",
    "    return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "spanish_ids = get_lang_ids(\"es\")\n",
    "print(\"# of Spanish Articles: \" + str(len(spanish_ids)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "for id1 in tqdm(spanish_ids):\n",
    "    if not os.path.exists(path + id1[:-4] + \"_trans.txt\"):\n",
    "        create_translation(path + id1, path + id1[:-4] + \"_trans.txt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create Slug dict and save"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "slugs_esen = {}\n",
    "for index, row in news_groups.iterrows():\n",
    "    slug = row['slug']\n",
    "    ids = row['filename']\n",
    "    lang = row['language']\n",
    "    if slug in slugs_esen:\n",
    "        slugs_esen[slug][lang] = ids\n",
    "    else:\n",
    "        slugs_esen[slug] = {}\n",
    "        slugs_esen[slug][lang] = ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "slugs_esen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "# Test set is created using the news articles(original and simplified) that have both spanish and english rewrite\n",
    "\n",
    "def create_test(k=0.8):\n",
    "    test_ids_es = []\n",
    "    test_ids_en = []\n",
    "    count=0\n",
    "    for slug in slugs_esen.keys():\n",
    "        if \"en\" in slugs_esen[slug] and \"es\" in slugs_esen[slug]:\n",
    "            count+=1\n",
    "            test_ids_es.extend(slugs_esen[slug][\"es\"])\n",
    "            test_ids_en.extend(slugs_esen[slug][\"en\"])\n",
    "    n=int(0.8*count)\n",
    "    return test_ids_es[:n], test_ids_es[n:], test_ids_en[:n], test_ids_en[-n:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_ids_es, dev_ids_es, test_ids_en, dev_ids_en = create_test()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_pairs():\n",
    "    es_pairs = []\n",
    "    en_pairs = []\n",
    "    esen_pairs = []\n",
    "    for slug in slugs_esen.keys():\n",
    "        if \"en\" in slugs_esen[slug]:\n",
    "            en_pairs.extend(list(itertools.combinations(slugs_esen[slug][\"en\"], 2)))\n",
    "        if \"es\" in slugs_esen[slug]:\n",
    "            es_pairs.extend(list(itertools.combinations(slugs_esen[slug][\"es\"], 2)))\n",
    "        if \"en\" in slugs_esen[slug] and \"es\" in slugs_esen[slug]:\n",
    "            esen_pairs.extend(list(itertools.product(slugs_esen[slug][\"es\"], slugs_esen[slug][\"en\"])))\n",
    "    return es_pairs, en_pairs, esen_pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "es_pairs, en_pairs, esen_pairs = create_pairs()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Also available in our CrossLingualAlignmentTool: https://github.com/sweta20/ComplexityControlledMT/tree/master/CrossLingualAlignment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "\"\"\" Path to Massalign \"\"\"\n",
    "sys.path.append(\"/usr/local/lib/python3.6/dist-packages/massalign\")\n",
    "\n",
    "from massalign.core import *\n",
    "from gach import sentence_align\n",
    "from nltk.tokenize.toktok import ToktokTokenizer\n",
    "toktok = ToktokTokenizer()\n",
    "\n",
    "m = MASSAligner()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_massalign_sentence_pairs(file1, file2):\n",
    "    #Train model over them:\n",
    "    model = TFIDFModel([file1, file2], 'https://ghpaetzold.github.io/massalign_data/stop_words.txt')\n",
    "    \n",
    "    #Get paragraph aligner:\n",
    "    paragraph_aligner = VicinityDrivenParagraphAligner(similarity_model=model, acceptable_similarity=0.3)\n",
    "\n",
    "    #Get sentence aligner:\n",
    "    sentence_aligner = VicinityDrivenSentenceAligner(similarity_model=model, acceptable_similarity=0.21, similarity_slack=0.05)\n",
    "\n",
    "    #Get paragraphs from the document:\n",
    "    p1s = m.getParagraphsFromDocument(file1)\n",
    "    p2s = m.getParagraphsFromDocument(file2)\n",
    "    #Align paragraphs:\n",
    "    alignments, aligned_paragraphs = m.getParagraphAlignments(p1s, p2s, paragraph_aligner)\n",
    "    \n",
    "    #Align sentences in each pair of aligned paragraphs:\n",
    "    alignmentsl = []\n",
    "    for a in aligned_paragraphs:\n",
    "        p1 = a[0]\n",
    "        p2 = a[1]\n",
    "        alignments, aligned_sentences = m.getSentenceAlignments(p1, p2, sentence_aligner)\n",
    "        \n",
    "        alignmentsl.extend(aligned_sentences)\n",
    "    return alignmentsl\n",
    "\n",
    "\n",
    "def create_spanish_english_alignments(id1, id2):\n",
    "    \n",
    "    spa_file = id1\n",
    "    eng_file = id2\n",
    "    spa_trans_file = id1[:-4] + \"_trans.txt\"\n",
    "    \n",
    "    massalign_sentence_pairs = get_massalign_sentence_pairs(spa_trans_file, eng_file)\n",
    "    translation_sentence_pairs = sentence_align(spa_file, spa_trans_file, 0.97, 1.8)\n",
    "    \n",
    "    pairs = []\n",
    "    for eng_trans, eng_org in massalign_sentence_pairs:\n",
    "        eng_simple_tok_1 = toktok.tokenize(eng_trans)\n",
    "        \n",
    "        spanish = ''\n",
    "        prev_spa = ''\n",
    "        for spa, eng in translation_sentence_pairs:\n",
    "            eng_simple_tok_2 = toktok.tokenize(eng)\n",
    "        \n",
    "            I = len(set(eng_simple_tok_2).intersection(set(eng_simple_tok_1)))\n",
    "            U = len(set(eng_simple_tok_2))\n",
    "            try:\n",
    "                percent_overlap = float(I)/U\n",
    "                if percent_overlap > 0.5 and spa!=prev_spa:\n",
    "                    spanish += spa\n",
    "                    prev_spa = spa\n",
    "                    break\n",
    "            except:\n",
    "                continue\n",
    "        if spanish != '':\n",
    "            pairs.append([spanish, eng_org])\n",
    "    return pairs\n",
    "\n",
    "def create_mono_alignments(id1, id2):\n",
    "    return get_massalign_sentence_pairs(id1, id2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "en_sentence_pairs = {}\n",
    "for pair in tqdm(en_pairs):\n",
    "    try:\n",
    "        en_sentence_pairs[pair] = create_mono_alignments(path + pair[0], path + pair[1])\n",
    "    except:\n",
    "        continue\n",
    "        \n",
    "lengths = {key:len(value) for key,value in en_sentence_pairs.items()}\n",
    "print(\"Sentence pairs\", sum(lengths.values()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"data/en_pairs.pkl\", \"wb\") as f:\n",
    "    pickle.dump(en_sentence_pairs, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "es_sentence_pairs = {}\n",
    "for pair in tqdm(es_pairs):\n",
    "    try:\n",
    "        es_sentence_pairs[pair] = create_mono_alignments(pair[0], pair[1])\n",
    "    except:\n",
    "        continue\n",
    "        \n",
    "lengths = {key:len(value) for key,value in es_sentence_pairs.items()}\n",
    "print(\"Sentence pairs\", sum(lengths.values()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"data/es_pairs.pkl\", \"wb\") as f:\n",
    "    pickle.dump(es_sentence_pairs, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "esen_sentence_pairs = {}\n",
    "for pair in tqdm(esen_pairs):\n",
    "    try:\n",
    "        esen_sentence_pairs[pair] = create_spanish_english_alignments(pair[0], pair[1])\n",
    "    except:\n",
    "        continue\n",
    "    \n",
    "lengths = {key:len(value) for key,value in esen_sentence_pairs.items()}\n",
    "print(\"Sentence pairs\", sum(lengths.values()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lengths = {key:len(value) for key,value in esen_sentence_pairs.items()}\n",
    "print(\"Sentence pairs\", sum(lengths.values()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "load all sentence pairs:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"data/en_pairs.pkl\", \"rb\") as f:\n",
    "    en_sentence_pairs = pickle.load(f)\n",
    "with open(\"data/es_pairs.pkl\", \"rb\") as f:\n",
    "    es_sentence_pairs = pickle.load(f)\n",
    "with open(\"data/esen_pairs.pkl\", \"rb\") as f:\n",
    "    esen_sentence_pairs = pickle.load(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create train/dev/test as reruired by the repository"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "token = {'2': '<TWO>', '3': '<THREE>' , '4': '<FOUR>', '5': '<FIVE>', '6' : '<SIX>',\n",
    "        '7': '<SEVEN>', '8':'<EIGHT>', '9' : '<NINE>', '10': '<TEN>', '11': '<ELEVEN>', '12' : '<TWELVE>'}\n",
    "inv_map = {v: k for k, v in token.items()}\n",
    "\n",
    "def write_to_file(data, split, f_prefix, path=\"data/\"):\n",
    "    src_file = open(path + split + \"_\" + f_prefix + \".src\", \"w\")\n",
    "    dst_file = open(path + split + \"_\" + f_prefix + \".tgt\", \"w\")\n",
    "    \n",
    "    for i in range(len(data)):\n",
    "        src_file.write(token[str(data[i][1])] + \"\\t\" +  data[i][2] + \"\\n\")\n",
    "        dst_file.write(data[i][3] + \"\\n\")\n",
    "    \n",
    "    src_file.close()\n",
    "    dst_file.close()\n",
    "    \n",
    "def create_train_test(sent_pairs, f_prefix, col=\"y\"):\n",
    "    train_data = []\n",
    "    test_data = []\n",
    "    dev_data = []\n",
    "    all_data = {}\n",
    "    \n",
    "    for pair in tqdm(sent_pairs):\n",
    "        id1, id2 = pair\n",
    "        y1 = data[data[\"id\"]==id1][col].values[0]\n",
    "        y2 = data[data[\"id\"]==id2][col].values[0]\n",
    "        if f_prefix == \"esen_neq\":\n",
    "            if int(y2) < int(y1):\n",
    "                sentences = sent_pairs[pair]\n",
    "                for sentence_pair in sentences:\n",
    "                    if (y1, y2) not in all_data:\n",
    "                        all_data[(y1, y2)] = 1\n",
    "                    else:\n",
    "                        all_data[(y1, y2)] += 1\n",
    "                    if id2 in test_ids_en:\n",
    "                        test_data.append((y1, y2, sentence_pair[0], sentence_pair[1]))\n",
    "                    elif id2 in dev_ids_en:\n",
    "                        dev_data.append((y1, y2, sentence_pair[0], sentence_pair[1]))\n",
    "                    else:\n",
    "                        train_data.append((y1, y2, sentence_pair[0], sentence_pair[1]))\n",
    "        elif f_prefix == \"esen_eq\":\n",
    "            if int(y2) == int(y1):\n",
    "                sentences = sent_pairs[pair]\n",
    "                for sentence_pair in sentences:\n",
    "                    if id2 in test_ids_en:\n",
    "                        test_data.append((y1, y2, sentence_pair[0], sentence_pair[1]))\n",
    "                    elif id2 in dev_ids_en:\n",
    "                        dev_data.append((y1, y2, sentence_pair[0], sentence_pair[1]))\n",
    "                    else:\n",
    "                        train_data.append((y1, y2, sentence_pair[0], sentence_pair[1]))\n",
    "        \n",
    "        elif f_prefix == \"es\":\n",
    "            sentences = sent_pairs[pair]\n",
    "            for sentence_pair in sentences:\n",
    "                if id2 in test_ids_es:\n",
    "                    test_data.append((y2, y1, sentence_pair[1], sentence_pair[0]))\n",
    "                elif id2 in dev_ids_es:\n",
    "                    dev_data.append((y2, y1, sentence_pair[1], sentence_pair[0]))\n",
    "                else:\n",
    "                     train_data.append((y2, y1, sentence_pair[1], sentence_pair[0]))\n",
    "    \n",
    "        else:\n",
    "            sentences = sent_pairs[pair]\n",
    "            for sentence_pair in sentences:\n",
    "                if (y2, y1) not in all_data:\n",
    "                    all_data[(y2, y1)] = 1\n",
    "                else:\n",
    "                    all_data[(y2, y1)] += 1\n",
    "                if id2 in test_ids_en:\n",
    "                    test_data.append((y2, y1, sentence_pair[1], sentence_pair[0]))\n",
    "                elif id2 in dev_ids_en:\n",
    "                    dev_data.append((y2, y1, sentence_pair[1], sentence_pair[0]))\n",
    "                else:\n",
    "                     train_data.append((y2, y1, sentence_pair[1], sentence_pair[0]))\n",
    "    \n",
    "    write_to_file(train_data, \"train\", f_prefix)\n",
    "    write_to_file(dev_data, \"dev\", f_prefix)\n",
    "    write_to_file(test_data, \"test\", f_prefix)\n",
    "    \n",
    "    print(\"# of train_data: {}, # of dev data: {}, # of test_data: {}\".format(len(train_data), len(dev_data), len(test_data)))\n",
    "    \n",
    "    return train_data, dev_data, test_data, all_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data, dev_data, test_data, all_data = create_train_test(en_sentence_pairs, \"en\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data, dev_data, test_data = create_train_test(es_sentence_pairs, \"es\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data, dev_data, test_data, all_data = create_train_test(esen_sentence_pairs, \"esen_neq\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data, dev_data, test_data = create_train_test(esen_sentence_pairs, \"esen_eq\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_grade_stats(file_name):\n",
    "    with open(file_name) as f:\n",
    "        data = f.readlines()\n",
    "    grade_stats = {}\n",
    "    for i in range(0,len(data)):\n",
    "        src = data[i].split(\"\\t\")[1]\n",
    "        tgt_grade = data[i].split(\"\\t\")[0]\n",
    "        if tgt_grade not in grade_stats:\n",
    "            grade_stats[tgt_grade] = 1\n",
    "        else:\n",
    "            grade_stats[tgt_grade] += 1\n",
    "            \n",
    "    return grade_stats, len(grade_stats)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grade_stats, _ = get_grade_stats(\"data/dev_es.src\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(3, 13):\n",
    "    for j in range(2,12):\n",
    "        if (i,j) in all_data:\n",
    "            print(i, j, all_data[(i,j)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Creating ARI dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append(\"/home/sweta/Work/Simplification/readability\")\n",
    "\n",
    "from compute_ari_accuracy import get_text_ari_grade_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clip_val(x, min_val=2, max_val=12):\n",
    "    if(x<min_val):\n",
    "        return min_val\n",
    "    elif(x>max_val):\n",
    "        return max_val\n",
    "    else:\n",
    "        return int(x)\n",
    "\n",
    "def get_ari_file(src_file, tgt_file, out_file):\n",
    "    src_data = open(src_file).readlines()\n",
    "    tgt_data = open(tgt_file).readlines()\n",
    "    \n",
    "    with open(out_file, \"w\") as f:\n",
    "        for i in range(len(src_data)):\n",
    "            tgt_ari_grade = token[str(clip_val(get_text_ari_grade_score(tgt_data[i].strip())))]\n",
    "            out_file.write(tgt_ari_grade + \"\\t\" + src_data.split(\"\\t\")[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "src_file = \"data/train_en.src\"\n",
    "tgt_file = \"data/train_en.tgt\"\n",
    "out_file = \"data/train_en_ari.src\"\n",
    "\n",
    "get_ari_file(src_file, tgt_file, out_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}