zero_shot_simple_roberta.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_t4OqVAvc0bp",
        "outputId": "6dcbda3c-cc20-40a7-d8da-43bbb72f09eb"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Cloning into 'SemEval_2022_Task2-idiomaticity'...\n",
            "remote: Enumerating objects: 92, done.\u001b[K\n",
            "remote: Counting objects: 100% (92/92), done.\u001b[K\n",
            "remote: Compressing objects: 100% (75/75), done.\u001b[K\n",
            "remote: Total 92 (delta 31), reused 67 (delta 15), pack-reused 0\u001b[K\n",
            "Unpacking objects: 100% (92/92), done.\n"
          ]
        }
      ],
      "source": [
        "!git clone https://github.com/H-TayyarMadabushi/SemEval_2022_Task2-idiomaticity.git"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tgOAbNz0c1fB",
        "outputId": "d3271f4b-b7e0-445e-db31-c097c47f36ce"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Cloning into 'AStitchInLanguageModels'...\n",
            "remote: Enumerating objects: 1030, done.\u001b[K\n",
            "remote: Counting objects: 100% (1030/1030), done.\u001b[K\n",
            "remote: Compressing objects: 100% (772/772), done.\u001b[K\n",
            "remote: Total 1030 (delta 382), reused 803 (delta 202), pack-reused 0\u001b[K\n",
            "Receiving objects: 100% (1030/1030), 79.86 MiB | 24.75 MiB/s, done.\n",
            "Resolving deltas: 100% (382/382), done.\n"
          ]
        }
      ],
      "source": [
        "!git clone https://github.com/H-TayyarMadabushi/AStitchInLanguageModels.git"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PDPx_vw6c1cG",
        "outputId": "ef17bd39-4126-4d3f-db53-01be8e98b54d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Cloning into 'transformers'...\n",
            "remote: Enumerating objects: 91488, done.\u001b[K\n",
            "remote: Counting objects: 100% (5/5), done.\u001b[K\n",
            "remote: Compressing objects: 100% (5/5), done.\u001b[K\n",
            "remote: Total 91488 (delta 0), reused 1 (delta 0), pack-reused 91483\u001b[K\n",
            "Receiving objects: 100% (91488/91488), 75.57 MiB | 18.72 MiB/s, done.\n",
            "Resolving deltas: 100% (66025/66025), done.\n",
            "/content/transformers\n",
            "Obtaining file:///content/transformers\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "    Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers==4.14.0.dev0) (21.3)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers==4.14.0.dev0) (1.19.5)\n",
            "Collecting pyyaml>=5.1\n",
            "  Downloading PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (596 kB)\n",
            "\u001b[K     |████████████████████████████████| 596 kB 13.6 MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers==4.14.0.dev0) (2019.12.20)\n",
            "Collecting huggingface-hub<1.0,>=0.1.0\n",
            "  Downloading huggingface_hub-0.2.1-py3-none-any.whl (61 kB)\n",
            "\u001b[K     |████████████████████████████████| 61 kB 516 kB/s \n",
            "\u001b[?25hRequirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers==4.14.0.dev0) (4.8.2)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers==4.14.0.dev0) (4.62.3)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers==4.14.0.dev0) (3.4.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers==4.14.0.dev0) (2.23.0)\n",
            "Collecting tokenizers<0.11,>=0.10.1\n",
            "  Downloading tokenizers-0.10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (3.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 3.3 MB 43.9 MB/s \n",
            "\u001b[?25hCollecting sacremoses\n",
            "  Downloading sacremoses-0.0.46-py3-none-any.whl (895 kB)\n",
            "\u001b[K     |████████████████████████████████| 895 kB 51.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.1.0->transformers==4.14.0.dev0) (3.10.0.2)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers==4.14.0.dev0) (3.0.6)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers==4.14.0.dev0) (3.6.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.14.0.dev0) (2021.10.8)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.14.0.dev0) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.14.0.dev0) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.14.0.dev0) (2.10)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==4.14.0.dev0) (1.1.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==4.14.0.dev0) (7.1.2)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==4.14.0.dev0) (1.15.0)\n",
            "Installing collected packages: pyyaml, tokenizers, sacremoses, huggingface-hub, transformers\n",
            "  Attempting uninstall: pyyaml\n",
            "    Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "  Running setup.py develop for transformers\n",
            "Successfully installed huggingface-hub-0.2.1 pyyaml-6.0 sacremoses-0.0.46 tokenizers-0.10.3 transformers-4.14.0.dev0\n",
            "/content\n"
          ]
        }
      ],
      "source": [
        "!git clone https://github.com/huggingface/transformers.git\n",
        "%cd transformers/\n",
        "!pip install --editable .\n",
        "%cd /content/ "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jFBG6zD0c1Yb",
        "outputId": "0d9396f1-7bbc-4147-8681-fb1c906b1a46"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Collecting datasets\n",
            "  Downloading datasets-1.16.1-py3-none-any.whl (298 kB)\n",
            "\u001b[?25l\r\u001b[K     |█                               | 10 kB 25.1 MB/s eta 0:00:01\r\u001b[K     |██▏                             | 20 kB 27.8 MB/s eta 0:00:01\r\u001b[K     |███▎                            | 30 kB 25.5 MB/s eta 0:00:01\r\u001b[K     |████▍                           | 40 kB 19.4 MB/s eta 0:00:01\r\u001b[K     |█████▌                          | 51 kB 15.4 MB/s eta 0:00:01\r\u001b[K     |██████▋                         | 61 kB 11.5 MB/s eta 0:00:01\r\u001b[K     |███████▊                        | 71 kB 12.2 MB/s eta 0:00:01\r\u001b[K     |████████▉                       | 81 kB 13.5 MB/s eta 0:00:01\r\u001b[K     |█████████▉                      | 92 kB 12.1 MB/s eta 0:00:01\r\u001b[K     |███████████                     | 102 kB 13.0 MB/s eta 0:00:01\r\u001b[K     |████████████                    | 112 kB 13.0 MB/s eta 0:00:01\r\u001b[K     |█████████████▏                  | 122 kB 13.0 MB/s eta 0:00:01\r\u001b[K     |██████████████▎                 | 133 kB 13.0 MB/s eta 0:00:01\r\u001b[K     |███████████████▍                | 143 kB 13.0 MB/s eta 0:00:01\r\u001b[K     |████████████████▌               | 153 kB 13.0 MB/s eta 0:00:01\r\u001b[K     |█████████████████▋              | 163 kB 13.0 MB/s eta 0:00:01\r\u001b[K     |██████████████████▊             | 174 kB 13.0 MB/s eta 0:00:01\r\u001b[K     |███████████████████▊            | 184 kB 13.0 MB/s eta 0:00:01\r\u001b[K     |████████████████████▉           | 194 kB 13.0 MB/s eta 0:00:01\r\u001b[K     |██████████████████████          | 204 kB 13.0 MB/s eta 0:00:01\r\u001b[K     |███████████████████████         | 215 kB 13.0 MB/s eta 0:00:01\r\u001b[K     |████████████████████████▏       | 225 kB 13.0 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▎      | 235 kB 13.0 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▍     | 245 kB 13.0 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▌    | 256 kB 13.0 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▋   | 266 kB 13.0 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▋  | 276 kB 13.0 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▊ | 286 kB 13.0 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▉| 296 kB 13.0 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 298 kB 13.0 MB/s \n",
            "\u001b[?25hRequirement already satisfied: huggingface-hub<1.0.0,>=0.1.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (0.2.1)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from datasets) (1.19.5)\n",
            "Collecting xxhash\n",
            "  Downloading xxhash-2.0.2-cp37-cp37m-manylinux2010_x86_64.whl (243 kB)\n",
            "\u001b[K     |████████████████████████████████| 243 kB 52.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: pyarrow!=4.0.0,>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (3.0.0)\n",
            "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.7/dist-packages (from datasets) (4.62.3)\n",
            "Requirement already satisfied: dill in /usr/local/lib/python3.7/dist-packages (from datasets) (0.3.4)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.7/dist-packages (from datasets) (0.70.12.2)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from datasets) (4.8.2)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (2.23.0)\n",
            "Collecting aiohttp\n",
            "  Downloading aiohttp-3.8.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (1.1 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1 MB 44.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from datasets) (21.3)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from datasets) (1.1.5)\n",
            "Collecting fsspec[http]>=2021.05.0\n",
            "  Downloading fsspec-2021.11.1-py3-none-any.whl (132 kB)\n",
            "\u001b[K     |████████████████████████████████| 132 kB 53.6 MB/s \n",
            "\u001b[?25hRequirement already satisfied: pyyaml in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (6.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (3.10.0.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (3.4.0)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->datasets) (3.0.6)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (2021.10.8)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (1.24.3)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (21.2.0)\n",
            "Collecting asynctest==0.13.0\n",
            "  Downloading asynctest-0.13.0-py3-none-any.whl (26 kB)\n",
            "Collecting frozenlist>=1.1.1\n",
            "  Downloading frozenlist-1.2.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (192 kB)\n",
            "\u001b[K     |████████████████████████████████| 192 kB 45.9 MB/s \n",
            "\u001b[?25hCollecting yarl<2.0,>=1.0\n",
            "  Downloading yarl-1.7.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (271 kB)\n",
            "\u001b[K     |████████████████████████████████| 271 kB 52.8 MB/s \n",
            "\u001b[?25hCollecting async-timeout<5.0,>=4.0.0a3\n",
            "  Downloading async_timeout-4.0.1-py3-none-any.whl (5.7 kB)\n",
            "Collecting multidict<7.0,>=4.5\n",
            "  Downloading multidict-5.2.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (160 kB)\n",
            "\u001b[K     |████████████████████████████████| 160 kB 49.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (2.0.8)\n",
            "Collecting aiosignal>=1.1.2\n",
            "  Downloading aiosignal-1.2.0-py3-none-any.whl (8.2 kB)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->datasets) (3.6.0)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2018.9)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas->datasets) (1.15.0)\n",
            "Installing collected packages: multidict, frozenlist, yarl, asynctest, async-timeout, aiosignal, fsspec, aiohttp, xxhash, datasets\n",
            "Successfully installed aiohttp-3.8.1 aiosignal-1.2.0 async-timeout-4.0.1 asynctest-0.13.0 datasets-1.16.1 frozenlist-1.2.0 fsspec-2021.11.1 multidict-5.2.0 xxhash-2.0.2 yarl-1.7.2\n"
          ]
        }
      ],
      "source": [
        "## run_glue needs this. \n",
        "!pip install datasets"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "s83pXR0Vc1Sn"
      },
      "outputs": [],
      "source": [
        "import site\n",
        "site.main()\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UiXPBZK2c1Ll"
      },
      "outputs": [],
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BbTHFE30aFfW"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "import csv\n",
        "\n",
        "from pathlib import Path"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-rUg4L14bWiD"
      },
      "outputs": [],
      "source": [
        "def load_csv( path, delimiter=',' ) : \n",
        "  header = None\n",
        "  data   = list()\n",
        "  with open( path, encoding='utf-8') as csvfile:\n",
        "    reader = csv.reader( csvfile, delimiter=delimiter ) \n",
        "    for row in reader : \n",
        "      if header is None : \n",
        "        header = row\n",
        "        continue\n",
        "      data.append( row ) \n",
        "  return header, data\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DnuRe4UxbXmT"
      },
      "outputs": [],
      "source": [
        "def write_csv( data, location ) : \n",
        "  with open( location, 'w', encoding='utf-8') as csvfile:\n",
        "    writer = csv.writer( csvfile ) \n",
        "    writer.writerows( data ) \n",
        "  print( \"Wrote {}\".format( location ) ) \n",
        "  return\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "IT6OFwo6bZIj"
      },
      "outputs": [],
      "source": [
        "def insert_to_submission_file( submission_format_file, input_file, prediction_format_file, setting ) :\n",
        "    submission_header, submission_content = load_csv( submission_format_file )\n",
        "    input_header     , input_data         = load_csv( input_file             )\n",
        "    prediction_header, prediction_data    = load_csv( prediction_format_file, '\\t' )\n",
        "\n",
        "    assert len( input_data ) == len( prediction_data )\n",
        "\n",
        "    ## submission_header ['ID', 'Language', 'Setting', 'Label']\n",
        "    ## input_header      ['label', 'sentence1' ]\n",
        "    ## prediction_header ['index', 'prediction']\n",
        "\n",
        "    prediction_data = list( reversed( prediction_data ) )\n",
        "\n",
        "    started_insert  = False\n",
        "    for elem in submission_content : \n",
        "        if elem[ submission_header.index( 'Setting' ) ] != setting :\n",
        "            if started_insert :\n",
        "                if len( prediction_data ) == 0 :\n",
        "                    break\n",
        "                else : \n",
        "                    raise Exception( \"Update should to contiguous ... something wrong.\" ) \n",
        "            continue\n",
        "        started_insert = True\n",
        "        elem[ submission_header.index( 'Label' ) ] = prediction_data.pop()[ prediction_header.index( 'prediction' ) ]\n",
        "\n",
        "    return [ submission_header ] + submission_content"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "igH5qaFXcWkN"
      },
      "outputs": [],
      "source": [
        "def _get_train_data( data_location, file_name, include_context, include_idiom ) :\n",
        "    \n",
        "    file_name = os.path.join( data_location, file_name ) \n",
        "\n",
        "    header, data = load_csv( file_name )\n",
        "\n",
        "    out_header = [ 'label', 'sentence1' ]\n",
        "    if include_idiom :\n",
        "        out_header = [ 'label', 'sentence1', 'sentence2' ]\n",
        "        \n",
        "    # ['DataID', 'Language', 'MWE', 'Setting', 'Previous', 'Target', 'Next', 'Label']\n",
        "    out_data = list()\n",
        "    for elem in data :\n",
        "        label     = elem[ header.index( 'Label'  ) ]\n",
        "        sentence1 = elem[ header.index( 'Target' ) ]\n",
        "        if include_context :\n",
        "            sentence1 = ' '.join( [ elem[ header.index( 'Previous' ) ], elem[ header.index( 'Target' ) ], elem[ header.index( 'Next' ) ] ] )\n",
        "        this_row = None\n",
        "        if not include_idiom :\n",
        "            this_row = [ label, sentence1 ] \n",
        "        else :\n",
        "            sentence2 = elem[ header.index( 'MWE' ) ]\n",
        "            this_row = [ label, sentence1, sentence2 ]\n",
        "        out_data.append( this_row )\n",
        "        assert len( out_header ) == len( this_row )\n",
        "    return [ out_header ] + out_data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4TEq7bEvcYeI"
      },
      "outputs": [],
      "source": [
        "def _get_dev_eval_data( data_location, input_file_name, gold_file_name, include_context, include_idiom ) :\n",
        "\n",
        "    input_headers, input_data = load_csv( os.path.join( data_location, input_file_name ) )\n",
        "    gold_header  = gold_data = None\n",
        "    if not gold_file_name is None : \n",
        "        gold_header  , gold_data  = load_csv( os.path.join( data_location, gold_file_name  ) )\n",
        "        assert len( input_data ) == len( gold_data )\n",
        "\n",
        "    # ['ID', 'Language', 'MWE', 'Previous', 'Target', 'Next']\n",
        "    # ['ID', 'DataID', 'Language', 'Label']\n",
        "    \n",
        "    out_header = [ 'label', 'sentence1' ]\n",
        "    if include_idiom :\n",
        "        out_header = [ 'label', 'sentence1', 'sentence2' ]\n",
        "\n",
        "    out_data = list()\n",
        "    for index in range( len( input_data ) ) :\n",
        "        label = 1\n",
        "        if not gold_file_name is None : \n",
        "            this_input_id = input_data[ index ][ input_headers.index( 'ID' ) ]\n",
        "            this_gold_id  = gold_data [ index ][ gold_header  .index( 'ID' ) ]\n",
        "            assert this_input_id == this_gold_id\n",
        "            \n",
        "            label     = gold_data[ index ][ gold_header.index( 'Label'  ) ]\n",
        "            \n",
        "        elem      = input_data[ index ]\n",
        "        sentence1 = elem[ input_headers.index( 'Target' ) ]\n",
        "        if include_context :\n",
        "            sentence1 = ' '.join( [ elem[ input_headers.index( 'Previous' ) ], elem[ input_headers.index( 'Target' ) ], elem[ input_headers.index( 'Next' ) ] ] )\n",
        "        this_row = None\n",
        "        if not include_idiom :\n",
        "            this_row = [ label, sentence1 ] \n",
        "        else :\n",
        "            sentence2 = elem[ input_headers.index( 'MWE' ) ]\n",
        "            this_row = [ label, sentence1, sentence2 ]\n",
        "        assert len( out_header ) == len( this_row ) \n",
        "        out_data.append( this_row )\n",
        "        \n",
        "\n",
        "    return [ out_header ] + out_data\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QLekZlxIcah1"
      },
      "outputs": [],
      "source": [
        "\"\"\"\n",
        "Based on the results presented in `AStitchInLanguageModels' we work with not including the idiom for the zero shot setting and including it in the one shot setting.\n",
        "\"\"\"\n",
        "def create_data( input_location, output_location ) :\n",
        "\n",
        "    \n",
        "    ## Zero shot data\n",
        "    train_data = _get_train_data(\n",
        "        data_location   = input_location,\n",
        "        file_name       = 'train_zero_shot.csv',\n",
        "        include_context = True,\n",
        "        include_idiom   = False\n",
        "    )\n",
        "    write_csv( train_data, os.path.join( output_location, 'ZeroShot', 'train.csv' ) )\n",
        "    \n",
        "    dev_data = _get_dev_eval_data(\n",
        "        data_location    = input_location,\n",
        "        input_file_name  = 'dev.csv',\n",
        "        gold_file_name   = 'dev_gold.csv', \n",
        "        include_context  = True,\n",
        "        include_idiom    = False\n",
        "    )        \n",
        "    write_csv( dev_data, os.path.join( output_location, 'ZeroShot', 'dev.csv' ) )\n",
        "    \n",
        "    eval_data = _get_dev_eval_data(\n",
        "        data_location    = input_location,\n",
        "        input_file_name  = 'eval.csv',\n",
        "        gold_file_name   = None , ## Don't have gold evaluation file -- submit to CodaLab\n",
        "        include_context  = True,\n",
        "        include_idiom    = False\n",
        "    )\n",
        "    write_csv( eval_data, os.path.join( output_location, 'ZeroShot', 'eval.csv' ) )\n",
        "\n",
        "\n",
        "    ## OneShot Data (combine both for training)\n",
        "    train_zero_data = _get_train_data(\n",
        "        data_location   = input_location,\n",
        "        file_name       = 'train_zero_shot.csv',\n",
        "        include_context = False,\n",
        "        include_idiom   = True\n",
        "    )\n",
        "    train_one_data = _get_train_data(\n",
        "        data_location   = input_location,\n",
        "        file_name       = 'train_one_shot.csv',\n",
        "        include_context = False,\n",
        "        include_idiom   = True\n",
        "    )\n",
        "\n",
        "    assert train_zero_data[0] == train_one_data[0] ## Headers\n",
        "    train_data = train_one_data + train_zero_data[1:]\n",
        "    write_csv( train_data, os.path.join( output_location, 'OneShot', 'train.csv' ) )\n",
        "    \n",
        "    dev_data = _get_dev_eval_data(\n",
        "        data_location    = input_location,\n",
        "        input_file_name  = 'dev.csv',\n",
        "        gold_file_name   = 'dev_gold.csv', \n",
        "        include_context  = False,\n",
        "        include_idiom    = True\n",
        "    )        \n",
        "    write_csv( dev_data, os.path.join( output_location, 'OneShot', 'dev.csv' ) )\n",
        "    \n",
        "    eval_data = _get_dev_eval_data(\n",
        "        data_location    = input_location,\n",
        "        input_file_name  = 'eval.csv',\n",
        "        gold_file_name   = None,\n",
        "        include_context  = False,\n",
        "        include_idiom    = True\n",
        "    )\n",
        "    write_csv( eval_data, os.path.join( output_location, 'OneShot', 'eval.csv' ) )\n",
        "\n",
        "    return"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TjUspjEdcumR",
        "outputId": "8ff5ad5f-6015-4fae-c392-ffd7af7d9987"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "AStitchInLanguageModels  SemEval_2022_Task2-idiomaticity\n",
            "sample_data\t\t transformers\n"
          ]
        }
      ],
      "source": [
        "!ls "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 996
        },
        "id": "OzWKqLy4cihp",
        "outputId": "72025f2a-3966-4e1e-dbab-390a676b2c61"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>DataID</th>\n",
              "      <th>Language</th>\n",
              "      <th>MWE</th>\n",
              "      <th>Setting</th>\n",
              "      <th>Previous</th>\n",
              "      <th>Target</th>\n",
              "      <th>Next</th>\n",
              "      <th>Label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>train_zero_shot.EN.168.1</td>\n",
              "      <td>EN</td>\n",
              "      <td>double dutch</td>\n",
              "      <td>zero_shot</td>\n",
              "      <td>This inspired others to jump ropes as a leisur...</td>\n",
              "      <td>There are several theories behind the origin o...</td>\n",
              "      <td>The most popular theory states that “Double Du...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>train_zero_shot.EN.168.2</td>\n",
              "      <td>EN</td>\n",
              "      <td>double dutch</td>\n",
              "      <td>zero_shot</td>\n",
              "      <td>In the age of chivalry a man paid for the woma...</td>\n",
              "      <td>Double Dutch also derives from the same era, D...</td>\n",
              "      <td>There are many phrases that include the word: ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>train_zero_shot.EN.168.3</td>\n",
              "      <td>EN</td>\n",
              "      <td>double dutch</td>\n",
              "      <td>zero_shot</td>\n",
              "      <td>To her eternal credit, she kept both India and...</td>\n",
              "      <td>Since 1977 we have had a plethora of Foreign M...</td>\n",
              "      <td>We need to exclude from that list the late Mr ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>train_zero_shot.EN.168.4</td>\n",
              "      <td>EN</td>\n",
              "      <td>double dutch</td>\n",
              "      <td>zero_shot</td>\n",
              "      <td>While pharmaceutical companies were researchin...</td>\n",
              "      <td>Turns out that these people were speaking doub...</td>\n",
              "      <td>So why aren’t Big Macs sold all over the world...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>train_zero_shot.EN.168.5</td>\n",
              "      <td>EN</td>\n",
              "      <td>double dutch</td>\n",
              "      <td>zero_shot</td>\n",
              "      <td>Coronavirus in Europe   * Brexit   * Brussels ...</td>\n",
              "      <td>Is Flemish premier talking double Dutch?</td>\n",
              "      <td>Three months before the Belgians take over the...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4486</th>\n",
              "      <td>train_zero_shot.PT.351.8</td>\n",
              "      <td>PT</td>\n",
              "      <td>gato-pingado</td>\n",
              "      <td>zero_shot</td>\n",
              "      <td>Era o ano de 1968.</td>\n",
              "      <td>A estação de passageiros tranquila, com um ou...</td>\n",
              "      <td>Com os primeiros raios solares, o potente DC 8...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4487</th>\n",
              "      <td>train_zero_shot.PT.351.9</td>\n",
              "      <td>PT</td>\n",
              "      <td>gato-pingado</td>\n",
              "      <td>zero_shot</td>\n",
              "      <td>De acordo com informações vazadas de dentro da...</td>\n",
              "      <td>Segundo a fonte, o programa vai seguir um form...</td>\n",
              "      <td>Leonardo fica inchado, perde a voz e vício mor...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4488</th>\n",
              "      <td>train_zero_shot.PT.351.10</td>\n",
              "      <td>PT</td>\n",
              "      <td>gato-pingado</td>\n",
              "      <td>zero_shot</td>\n",
              "      <td>O radialista Roberto Toledo apresentou o show ...</td>\n",
              "      <td>\"Eu fiquei sentado no chão e realmente só tinh...</td>\n",
              "      <td>Entre os poucos espectadores estava o advogado...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4489</th>\n",
              "      <td>train_zero_shot.PT.351.11</td>\n",
              "      <td>PT</td>\n",
              "      <td>gato-pingado</td>\n",
              "      <td>zero_shot</td>\n",
              "      <td>O carnaval em Goiânia também tem espaço para o...</td>\n",
              "      <td>É o caso do bloco “Gato Pingado”, que reuniu c...</td>\n",
              "      <td>Segundo os organizadores, quando criaram o blo...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4490</th>\n",
              "      <td>train_zero_shot.PT.351.12</td>\n",
              "      <td>PT</td>\n",
              "      <td>gato-pingado</td>\n",
              "      <td>zero_shot</td>\n",
              "      <td>A concentração aconteceu na Avenida Circular, ...</td>\n",
              "      <td>De acordo com presidente do bloco, Paulo de Tá...</td>\n",
              "      <td>“A diferença do bloco é que realmente é um ca...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>4491 rows × 8 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                         DataID  ... Label\n",
              "0      train_zero_shot.EN.168.1  ...     0\n",
              "1      train_zero_shot.EN.168.2  ...     0\n",
              "2      train_zero_shot.EN.168.3  ...     0\n",
              "3      train_zero_shot.EN.168.4  ...     0\n",
              "4      train_zero_shot.EN.168.5  ...     0\n",
              "...                         ...  ...   ...\n",
              "4486   train_zero_shot.PT.351.8  ...     0\n",
              "4487   train_zero_shot.PT.351.9  ...     0\n",
              "4488  train_zero_shot.PT.351.10  ...     0\n",
              "4489  train_zero_shot.PT.351.11  ...     1\n",
              "4490  train_zero_shot.PT.351.12  ...     1\n",
              "\n",
              "[4491 rows x 8 columns]"
            ]
          },
          "execution_count": 14,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "import pandas as pd\n",
        "df = pd.read_csv(\"/content/SemEval_2022_Task2-idiomaticity/SubTaskA/Data/train_zero_shot.csv\")\n",
        "df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 927
        },
        "id": "i-1uJyULckYH",
        "outputId": "f45d99dc-e72c-44f2-efbf-9afce46eba18"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>DataID</th>\n",
              "      <th>Language</th>\n",
              "      <th>MWE</th>\n",
              "      <th>Setting</th>\n",
              "      <th>Previous</th>\n",
              "      <th>Target</th>\n",
              "      <th>Next</th>\n",
              "      <th>Label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>train_one_shot.EN.147.1</td>\n",
              "      <td>EN</td>\n",
              "      <td>high life</td>\n",
              "      <td>one_shot</td>\n",
              "      <td>PSG ace Marquinhos owns this £170k Ferrari - b...</td>\n",
              "      <td>Despite having the riches to afford the high l...</td>\n",
              "      <td>In an interview with the French magazine 'So F...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>train_one_shot.EN.183.1</td>\n",
              "      <td>EN</td>\n",
              "      <td>birth rate</td>\n",
              "      <td>one_shot</td>\n",
              "      <td>Demography expert Piotr Szukalski told Dzienni...</td>\n",
              "      <td>Minister of Family and Social Policy Marlena M...</td>\n",
              "      <td>Commenting on data the state agency Statistics...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>train_one_shot.EN.213.1</td>\n",
              "      <td>EN</td>\n",
              "      <td>home run</td>\n",
              "      <td>one_shot</td>\n",
              "      <td>Negro Leagues Baseball Museum President Bob Ke...</td>\n",
              "      <td>So Aaron faced the same brutal racism other Bl...</td>\n",
              "      <td>This Black man in the deep South was about to ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>train_one_shot.EN.26.1</td>\n",
              "      <td>EN</td>\n",
              "      <td>public service</td>\n",
              "      <td>one_shot</td>\n",
              "      <td>The Nakayama Public Service Scholarship is par...</td>\n",
              "      <td>Program leaders said the scholarship defines p...</td>\n",
              "      <td>I think everyone can agree we need to do more ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>train_one_shot.EN.26.2</td>\n",
              "      <td>EN</td>\n",
              "      <td>public service</td>\n",
              "      <td>one_shot</td>\n",
              "      <td>He was absolutely right.”</td>\n",
              "      <td>In the ensuing years, Wennberg might not have ...</td>\n",
              "      <td>Wennberg said he knew when he ran for his sixt...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>135</th>\n",
              "      <td>train_one_shot.PT.293.1</td>\n",
              "      <td>PT</td>\n",
              "      <td>amigo oculto</td>\n",
              "      <td>one_shot</td>\n",
              "      <td>Mesmo com a pandemia, é possível manter a trad...</td>\n",
              "      <td>Chegou o fim do ano e com ele as festas de Nat...</td>\n",
              "      <td>Não tem jeito, essa é uma tradição brasileira.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>136</th>\n",
              "      <td>train_one_shot.PT.293.2</td>\n",
              "      <td>PT</td>\n",
              "      <td>amigo oculto</td>\n",
              "      <td>one_shot</td>\n",
              "      <td>Secret Santa Generator : Secret Santa é o nome...</td>\n",
              "      <td>Esta é a versão do app semelhante ao Amigo Ocu...</td>\n",
              "      <td>Eles são análogos e funcionam exatamente da me...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>137</th>\n",
              "      <td>train_one_shot.PT.284.1</td>\n",
              "      <td>PT</td>\n",
              "      <td>agente secreto</td>\n",
              "      <td>one_shot</td>\n",
              "      <td>Já à meia-noite, o Domingo Maior exibe 3 Dias ...</td>\n",
              "      <td>Ethan Renner (Kevin Costner), veterano agente ...</td>\n",
              "      <td>Seu último desejo é reatar com sua filha, com ...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>138</th>\n",
              "      <td>train_one_shot.PT.435.1</td>\n",
              "      <td>PT</td>\n",
              "      <td>sangue frio</td>\n",
              "      <td>one_shot</td>\n",
              "      <td>Com a pele em chamas, o companheiro de Khen (P...</td>\n",
              "      <td>Irritado com a mentira, o líder desejará matar...</td>\n",
              "      <td>Assim, ele decidirá se livrar do escolhido e l...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>139</th>\n",
              "      <td>train_one_shot.PT.435.2</td>\n",
              "      <td>PT</td>\n",
              "      <td>sangue frio</td>\n",
              "      <td>one_shot</td>\n",
              "      <td>O juiz Dalton Igor Kita Conrado, da 5ª Vara Fe...</td>\n",
              "      <td>A denúncia, que também implica Nilza dos Santo...</td>\n",
              "      <td>Ingressada em 2018, a denúncia do MPF, assinad...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>140 rows × 8 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                      DataID  ... Label\n",
              "0    train_one_shot.EN.147.1  ...     1\n",
              "1    train_one_shot.EN.183.1  ...     1\n",
              "2    train_one_shot.EN.213.1  ...     0\n",
              "3     train_one_shot.EN.26.1  ...     0\n",
              "4     train_one_shot.EN.26.2  ...     1\n",
              "..                       ...  ...   ...\n",
              "135  train_one_shot.PT.293.1  ...     0\n",
              "136  train_one_shot.PT.293.2  ...     1\n",
              "137  train_one_shot.PT.284.1  ...     1\n",
              "138  train_one_shot.PT.435.1  ...     0\n",
              "139  train_one_shot.PT.435.2  ...     1\n",
              "\n",
              "[140 rows x 8 columns]"
            ]
          },
          "execution_count": 15,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df_one = pd.read_csv(\"/content/SemEval_2022_Task2-idiomaticity/SubTaskA/Data/train_one_shot.csv\")\n",
        "df_one"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CLPsQFgrdE7A",
        "outputId": "7adbbe6b-6aff-412e-fecc-78f968c4c6ae"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Wrote Data/ZeroShot/train.csv\n",
            "Wrote Data/ZeroShot/dev.csv\n",
            "Wrote Data/ZeroShot/eval.csv\n",
            "Wrote Data/OneShot/train.csv\n",
            "Wrote Data/OneShot/dev.csv\n",
            "Wrote Data/OneShot/eval.csv\n"
          ]
        }
      ],
      "source": [
        "outpath = 'Data'\n",
        "    \n",
        "Path( os.path.join( outpath, 'ZeroShot' ) ).mkdir(parents=True, exist_ok=True)\n",
        "Path( os.path.join( outpath, 'OneShot' ) ).mkdir(parents=True, exist_ok=True)\n",
        "\n",
        "create_data( 'SemEval_2022_Task2-idiomaticity/SubTaskA/Data/', outpath )"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rFN4xuROkneH"
      },
      "outputs": [],
      "source": [
        "# model_config = BartConfig.from_pretrained(pretrain_model_dir) model_config.num_labels=new_num_labels model = BartForSequenceClassification.from_pretrained(pretrain_model_dir, config=model_config)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/"
        },
        "id": "t69_0dobdzoN",
        "outputId": "d92ffbcf-1f38-4e44-9010-765876cbdd06"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "12/11/2021 09:14:20 - WARNING - __main__ -   Process rank: -1, device: cpu, n_gpu: 0distributed training: False, 16-bits training: False\n",
            "12/11/2021 09:14:20 - INFO - __main__ -   Training/evaluation parameters TrainingArguments(\n",
            "_n_gpu=0,\n",
            "adafactor=False,\n",
            "adam_beta1=0.9,\n",
            "adam_beta2=0.999,\n",
            "adam_epsilon=1e-08,\n",
            "bf16=False,\n",
            "bf16_full_eval=False,\n",
            "dataloader_drop_last=False,\n",
            "dataloader_num_workers=0,\n",
            "dataloader_pin_memory=True,\n",
            "ddp_find_unused_parameters=None,\n",
            "debug=[],\n",
            "deepspeed=None,\n",
            "disable_tqdm=False,\n",
            "do_eval=True,\n",
            "do_predict=False,\n",
            "do_train=True,\n",
            "eval_accumulation_steps=None,\n",
            "eval_steps=None,\n",
            "evaluation_strategy=IntervalStrategy.EPOCH,\n",
            "fp16=False,\n",
            "fp16_backend=auto,\n",
            "fp16_full_eval=False,\n",
            "fp16_opt_level=O1,\n",
            "gradient_accumulation_steps=1,\n",
            "gradient_checkpointing=False,\n",
            "greater_is_better=True,\n",
            "group_by_length=False,\n",
            "half_precision_backend=auto,\n",
            "hub_model_id=None,\n",
            "hub_strategy=HubStrategy.EVERY_SAVE,\n",
            "hub_token=<HUB_TOKEN>,\n",
            "ignore_data_skip=False,\n",
            "label_names=None,\n",
            "label_smoothing_factor=0.0,\n",
            "learning_rate=2e-05,\n",
            "length_column_name=length,\n",
            "load_best_model_at_end=True,\n",
            "local_rank=-1,\n",
            "log_level=-1,\n",
            "log_level_replica=-1,\n",
            "log_on_each_node=True,\n",
            "logging_dir=models/ZeroShot/1/runs/Dec11_09-14-20_ea9c4ef026fa,\n",
            "logging_first_step=False,\n",
            "logging_nan_inf_filter=True,\n",
            "logging_steps=500,\n",
            "logging_strategy=IntervalStrategy.STEPS,\n",
            "lr_scheduler_type=SchedulerType.LINEAR,\n",
            "max_grad_norm=1.0,\n",
            "max_steps=-1,\n",
            "metric_for_best_model=f1,\n",
            "mp_parameters=,\n",
            "no_cuda=False,\n",
            "num_train_epochs=9.0,\n",
            "output_dir=models/ZeroShot/1/,\n",
            "overwrite_output_dir=False,\n",
            "past_index=-1,\n",
            "per_device_eval_batch_size=8,\n",
            "per_device_train_batch_size=32,\n",
            "prediction_loss_only=False,\n",
            "push_to_hub=False,\n",
            "push_to_hub_model_id=None,\n",
            "push_to_hub_organization=None,\n",
            "push_to_hub_token=<PUSH_TO_HUB_TOKEN>,\n",
            "remove_unused_columns=True,\n",
            "report_to=['tensorboard'],\n",
            "resume_from_checkpoint=None,\n",
            "run_name=models/ZeroShot/1/,\n",
            "save_on_each_node=False,\n",
            "save_steps=500,\n",
            "save_strategy=IntervalStrategy.EPOCH,\n",
            "save_total_limit=1,\n",
            "seed=0,\n",
            "sharded_ddp=[],\n",
            "skip_memory_metrics=True,\n",
            "tf32=None,\n",
            "tpu_metrics_debug=False,\n",
            "tpu_num_cores=None,\n",
            "use_legacy_prediction_loop=False,\n",
            "warmup_ratio=0.0,\n",
            "warmup_steps=0,\n",
            "weight_decay=0.0,\n",
            "xpu_backend=None,\n",
            ")\n",
            "12/11/2021 09:14:20 - INFO - __main__ -   load a local file for train: Data/ZeroShot/train.csv\n",
            "12/11/2021 09:14:20 - INFO - __main__ -   load a local file for validation: Data/ZeroShot/dev.csv\n",
            "12/11/2021 09:14:21 - WARNING - datasets.builder -   Using custom data configuration default-fc826c8dfede28b8\n",
            "12/11/2021 09:14:21 - WARNING - datasets.builder -   Reusing dataset csv (/root/.cache/huggingface/datasets/csv/default-fc826c8dfede28b8/0.0.0/bf68a4c4aefa545d0712b2fcbb1b327f905bbe2f6425fbc5e8c25234acb9e14a)\n",
            "100% 2/2 [00:00<00:00, 917.19it/s]\n",
            "[INFO|file_utils.py:1887] 2021-12-11 09:14:21,920 >> https://huggingface.co/roberta-base/resolve/main/config.json not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmpi7ekiv8y\n",
            "Downloading: 100% 481/481 [00:00<00:00, 524kB/s]\n",
            "[INFO|file_utils.py:1891] 2021-12-11 09:14:22,283 >> storing https://huggingface.co/roberta-base/resolve/main/config.json in cache at /root/.cache/huggingface/transformers/733bade19e5f0ce98e6531021dd5180994bb2f7b8bd7e80c7968805834ba351e.35205c6cfc956461d8515139f0f8dd5d207a2f336c0c3a83b4bc8dca3518e37b\n",
            "[INFO|file_utils.py:1899] 2021-12-11 09:14:22,283 >> creating metadata file for /root/.cache/huggingface/transformers/733bade19e5f0ce98e6531021dd5180994bb2f7b8bd7e80c7968805834ba351e.35205c6cfc956461d8515139f0f8dd5d207a2f336c0c3a83b4bc8dca3518e37b\n",
            "[INFO|configuration_utils.py:604] 2021-12-11 09:14:22,283 >> loading configuration file https://huggingface.co/roberta-base/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/733bade19e5f0ce98e6531021dd5180994bb2f7b8bd7e80c7968805834ba351e.35205c6cfc956461d8515139f0f8dd5d207a2f336c0c3a83b4bc8dca3518e37b\n",
            "[INFO|configuration_utils.py:641] 2021-12-11 09:14:22,285 >> Model config RobertaConfig {\n",
            "  \"_name_or_path\": \"roberta-base\",\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.14.0.dev0\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "[INFO|tokenization_auto.py:352] 2021-12-11 09:14:22,650 >> Could not locate the tokenizer configuration file, will try to use the model config instead.\n",
            "[INFO|configuration_utils.py:604] 2021-12-11 09:14:23,381 >> loading configuration file https://huggingface.co/roberta-base/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/733bade19e5f0ce98e6531021dd5180994bb2f7b8bd7e80c7968805834ba351e.35205c6cfc956461d8515139f0f8dd5d207a2f336c0c3a83b4bc8dca3518e37b\n",
            "[INFO|configuration_utils.py:641] 2021-12-11 09:14:23,381 >> Model config RobertaConfig {\n",
            "  \"_name_or_path\": \"roberta-base\",\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.14.0.dev0\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "[INFO|file_utils.py:1887] 2021-12-11 09:14:24,113 >> https://huggingface.co/roberta-base/resolve/main/vocab.json not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmpp1f9vtag\n",
            "Downloading: 100% 878k/878k [00:00<00:00, 1.76MB/s]\n",
            "[INFO|file_utils.py:1891] 2021-12-11 09:14:24,995 >> storing https://huggingface.co/roberta-base/resolve/main/vocab.json in cache at /root/.cache/huggingface/transformers/d3ccdbfeb9aaa747ef20432d4976c32ee3fa69663b379deb253ccfce2bb1fdc5.d67d6b367eb24ab43b08ad55e014cf254076934f71d832bbab9ad35644a375ab\n",
            "[INFO|file_utils.py:1899] 2021-12-11 09:14:24,995 >> creating metadata file for /root/.cache/huggingface/transformers/d3ccdbfeb9aaa747ef20432d4976c32ee3fa69663b379deb253ccfce2bb1fdc5.d67d6b367eb24ab43b08ad55e014cf254076934f71d832bbab9ad35644a375ab\n",
            "[INFO|file_utils.py:1887] 2021-12-11 09:14:25,361 >> https://huggingface.co/roberta-base/resolve/main/merges.txt not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmpzpl1jvve\n",
            "Downloading: 100% 446k/446k [00:00<00:00, 1.07MB/s]\n",
            "[INFO|file_utils.py:1891] 2021-12-11 09:14:26,154 >> storing https://huggingface.co/roberta-base/resolve/main/merges.txt in cache at /root/.cache/huggingface/transformers/cafdecc90fcab17011e12ac813dd574b4b3fea39da6dd817813efa010262ff3f.5d12962c5ee615a4c803841266e9c3be9a691a924f72d395d3a6c6c81157788b\n",
            "[INFO|file_utils.py:1899] 2021-12-11 09:14:26,155 >> creating metadata file for /root/.cache/huggingface/transformers/cafdecc90fcab17011e12ac813dd574b4b3fea39da6dd817813efa010262ff3f.5d12962c5ee615a4c803841266e9c3be9a691a924f72d395d3a6c6c81157788b\n",
            "[INFO|file_utils.py:1887] 2021-12-11 09:14:26,519 >> https://huggingface.co/roberta-base/resolve/main/tokenizer.json not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmpt5u1wi17\n",
            "Downloading: 100% 1.29M/1.29M [00:00<00:00, 2.61MB/s]\n",
            "[INFO|file_utils.py:1891] 2021-12-11 09:14:27,417 >> storing https://huggingface.co/roberta-base/resolve/main/tokenizer.json in cache at /root/.cache/huggingface/transformers/d53fc0fa09b8342651efd4073d75e19617b3e51287c2a535becda5808a8db287.fc9576039592f026ad76a1c231b89aee8668488c671dfbe6616bab2ed298d730\n",
            "[INFO|file_utils.py:1899] 2021-12-11 09:14:27,417 >> creating metadata file for /root/.cache/huggingface/transformers/d53fc0fa09b8342651efd4073d75e19617b3e51287c2a535becda5808a8db287.fc9576039592f026ad76a1c231b89aee8668488c671dfbe6616bab2ed298d730\n",
            "[INFO|tokenization_utils_base.py:1742] 2021-12-11 09:14:28,539 >> loading file https://huggingface.co/roberta-base/resolve/main/vocab.json from cache at /root/.cache/huggingface/transformers/d3ccdbfeb9aaa747ef20432d4976c32ee3fa69663b379deb253ccfce2bb1fdc5.d67d6b367eb24ab43b08ad55e014cf254076934f71d832bbab9ad35644a375ab\n",
            "[INFO|tokenization_utils_base.py:1742] 2021-12-11 09:14:28,539 >> loading file https://huggingface.co/roberta-base/resolve/main/merges.txt from cache at /root/.cache/huggingface/transformers/cafdecc90fcab17011e12ac813dd574b4b3fea39da6dd817813efa010262ff3f.5d12962c5ee615a4c803841266e9c3be9a691a924f72d395d3a6c6c81157788b\n",
            "[INFO|tokenization_utils_base.py:1742] 2021-12-11 09:14:28,539 >> loading file https://huggingface.co/roberta-base/resolve/main/tokenizer.json from cache at /root/.cache/huggingface/transformers/d53fc0fa09b8342651efd4073d75e19617b3e51287c2a535becda5808a8db287.fc9576039592f026ad76a1c231b89aee8668488c671dfbe6616bab2ed298d730\n",
            "[INFO|tokenization_utils_base.py:1742] 2021-12-11 09:14:28,539 >> loading file https://huggingface.co/roberta-base/resolve/main/added_tokens.json from cache at None\n",
            "[INFO|tokenization_utils_base.py:1742] 2021-12-11 09:14:28,539 >> loading file https://huggingface.co/roberta-base/resolve/main/special_tokens_map.json from cache at None\n",
            "[INFO|tokenization_utils_base.py:1742] 2021-12-11 09:14:28,539 >> loading file https://huggingface.co/roberta-base/resolve/main/tokenizer_config.json from cache at None\n",
            "[INFO|configuration_utils.py:604] 2021-12-11 09:14:29,268 >> loading configuration file https://huggingface.co/roberta-base/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/733bade19e5f0ce98e6531021dd5180994bb2f7b8bd7e80c7968805834ba351e.35205c6cfc956461d8515139f0f8dd5d207a2f336c0c3a83b4bc8dca3518e37b\n",
            "[INFO|configuration_utils.py:641] 2021-12-11 09:14:29,268 >> Model config RobertaConfig {\n",
            "  \"_name_or_path\": \"roberta-base\",\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.14.0.dev0\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "[INFO|file_utils.py:1887] 2021-12-11 09:14:29,759 >> https://huggingface.co/roberta-base/resolve/main/pytorch_model.bin not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmpehxgvbn1\n",
            "Downloading: 100% 478M/478M [00:11<00:00, 44.2MB/s]\n",
            "[INFO|file_utils.py:1891] 2021-12-11 09:14:41,160 >> storing https://huggingface.co/roberta-base/resolve/main/pytorch_model.bin in cache at /root/.cache/huggingface/transformers/51ba668f7ff34e7cdfa9561e8361747738113878850a7d717dbc69de8683aaad.c7efaa30a0d80b2958b876969faa180e485944a849deee4ad482332de65365a7\n",
            "[INFO|file_utils.py:1899] 2021-12-11 09:14:41,160 >> creating metadata file for /root/.cache/huggingface/transformers/51ba668f7ff34e7cdfa9561e8361747738113878850a7d717dbc69de8683aaad.c7efaa30a0d80b2958b876969faa180e485944a849deee4ad482332de65365a7\n",
            "[INFO|modeling_utils.py:1352] 2021-12-11 09:14:41,161 >> loading weights file https://huggingface.co/roberta-base/resolve/main/pytorch_model.bin from cache at /root/.cache/huggingface/transformers/51ba668f7ff34e7cdfa9561e8361747738113878850a7d717dbc69de8683aaad.c7efaa30a0d80b2958b876969faa180e485944a849deee4ad482332de65365a7\n",
            "[WARNING|modeling_utils.py:1611] 2021-12-11 09:14:43,168 >> Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['roberta.pooler.dense.bias', 'lm_head.bias', 'lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.weight', 'lm_head.decoder.weight', 'lm_head.dense.bias', 'lm_head.layer_norm.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "[WARNING|modeling_utils.py:1622] 2021-12-11 09:14:43,168 >> Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "100% 5/5 [00:01<00:00,  4.71ba/s]\n",
            "100% 1/1 [00:00<00:00,  6.18ba/s]\n",
            "12/11/2021 09:14:44 - INFO - __main__ -   Sample 3155 of the training set: {'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'input_ids': [0, 14693, 7, 750, 4, 175, 6, 5, 12863, 183, 21, 3249, 2967, 30, 10377, 30339, 6, 53, 42, 6953, 222, 45, 1338, 5, 4669, 232, 454, 20487, 28042, 17, 27, 29, 11604, 11, 2248, 9543, 4, 28042, 172, 1412, 5, 12863, 76, 7127, 7, 4190, 5, 936, 6, 61, 21, 423, 15517, 11, 10753, 19, 92, 2655, 59, 5, 6872, 17, 27, 29, 13435, 30, 8509, 12006, 88, 5, 4275, 23943, 7127, 14, 52, 14095, 452, 4, 1927, 4, 1132, 2594, 358, 237, 107, 6, 142, 5, 6872, 12684, 3441, 19671, 4, 1244, 360, 7, 1498, 63, 13435, 198, 5, 3778, 4, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'label': 0, 'sentence1': 'According to history.com, the leap day was originally discovered by Egyptian astronomers, but this discovery did not reach the western world until Julius Caesar’s reign in 45 BC. Caesar then created the leap year calendar to fix the problem, which was later adapted in accordance with new knowledge about the earth’s orbit by Pope Gregory into the Gregorian calendar that we observe today. Feb. 29 happens every four years, because the earth technically requires 365.25 days to complete its orbit around the sun.'}.\n",
            "12/11/2021 09:14:44 - INFO - __main__ -   Sample 3445 of the training set: {'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'input_ids': [0, 250, 2241, 1977, 6106, 2955, 3977, 1496, 6, 842, 571, 20870, 163, 7374, 2716, 6, 6723, 242, 263, 10, 3381, 3849, 8906, 293, 1192, 821, 19112, 424, 24200, 102, 3840, 11988, 475, 5655, 181, 2413, 1535, 364, 18721, 1243, 3840, 7, 19202, 208, 4214, 15333, 126, 726, 4349, 3137, 3006, 1526, 6374, 3840, 11988, 24182, 405, 366, 3861, 366, 2955, 9171, 10071, 3381, 4214, 385, 281, 7300, 3381, 3849, 8906, 293, 1890, 19926, 36, 2191, 791, 238, 10, 1931, 12, 10443, 102, 109, 9614, 6145, 163, 7374, 2716, 7745, 1438, 11738, 16738, 263, 6044, 1977, 24151, 181, 2413, 1535, 364, 24421, 366, 1192, 7996, 16384, 181, 6658, 37895, 493, 2841, 5030, 329, 263, 12206, 2137, 10, 2241, 5874, 2794, 2955, 3495, 5571, 3381, 4214, 4, 384, 2], 'label': 1, 'sentence1': 'A saída da crise, segundo Bachelet, depende de ações que garantam renda para os mais pobres e vacina para todos São Paulo – Alta comissária para os Direitos Humanos da Organização das Nações Unidas (ONU), a ex-presidenta do Chile Michelle Bachelet criticou governantes de países pobres e ricos que optaram pela economia em vez de promover a saúde da população. O resultado, segundo ela, foi o aprofundamento das desigualdades sociais causadas pela histórica falta de investimento em áreas sociais, entre elas a saúde.'}.\n",
            "12/11/2021 09:14:44 - INFO - __main__ -   Sample 331 of the training set: {'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'input_ids': [0, 9157, 16771, 21389, 14828, 6, 11, 3088, 19, 272, 306, 104, 6, 33, 2226, 10, 92, 169, 9, 5261, 1055, 8, 8528, 136, 1912, 77, 27991, 6234, 20432, 18, 4, 374, 392, 365, 212, 23, 5, 38, 7881, 3586, 2010, 5354, 4229, 2338, 6, 32253, 21389, 14828, 58, 16962, 19, 5, 22, 46577, 2010, 7702, 9, 5, 2041, 113, 2354, 6, 13, 5, 8137, 21389, 14828, 4, 20, 2844, 21, 2964, 23, 5, 8353, 12700, 4369, 6884, 4104, 4, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'label': 1, 'sentence1': 'Concept Smoke Screen, in partnership with G4S, have developed a new way of defending cash and guards against attacks when replenishing ATM\\'s. On May 11th at the IFSEC Security Industry Awards 2009, Concept Smoke Screen were honoured with the \"Physical Security Product of the Year\" award, for the Guardian Smoke Screen. The ceremony was conducted at the Birmingham Hilton Metropole.'}.\n",
            "[INFO|trainer.py:549] 2021-12-11 09:14:44,597 >> The following columns in the training set  don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: sentence1.\n",
            "[INFO|trainer.py:1204] 2021-12-11 09:14:44,618 >> ***** Running training *****\n",
            "[INFO|trainer.py:1205] 2021-12-11 09:14:44,618 >>   Num examples = 4491\n",
            "[INFO|trainer.py:1206] 2021-12-11 09:14:44,618 >>   Num Epochs = 9\n",
            "[INFO|trainer.py:1207] 2021-12-11 09:14:44,618 >>   Instantaneous batch size per device = 32\n",
            "[INFO|trainer.py:1208] 2021-12-11 09:14:44,618 >>   Total train batch size (w. parallel, distributed & accumulation) = 32\n",
            "[INFO|trainer.py:1209] 2021-12-11 09:14:44,618 >>   Gradient Accumulation steps = 1\n",
            "[INFO|trainer.py:1210] 2021-12-11 09:14:44,618 >>   Total optimization steps = 1269\n",
            "  3% 35/1269 [58:00<33:56:05, 99.00s/it]"
          ]
        }
      ],
      "source": [
        "!python /content/AStitchInLanguageModels/Dataset/Task2/Utils/run_glue_f1_macro.py \\\n",
        "    \t--model_name_or_path 'roberta-base' \\\n",
        "    \t--do_train \\\n",
        "    \t--do_eval \\\n",
        "    \t--max_seq_length 128 \\\n",
        "    \t--per_device_train_batch_size 32 \\\n",
        "    \t--learning_rate 2e-5 \\\n",
        "    \t--num_train_epochs 9 \\\n",
        "    \t--evaluation_strategy \"epoch\" \\\n",
        "    \t--output_dir models/ZeroShot/1/ \\\n",
        "    \t--seed 0 \\\n",
        "    \t--train_file      Data/ZeroShot/train.csv \\\n",
        "    \t--validation_file Data/ZeroShot/dev.csv \\\n",
        "\t    --evaluation_strategy \"epoch\" \\\n",
        "\t    --save_strategy \"epoch\"  \\\n",
        "\t    --load_best_model_at_end \\\n",
        "\t    --metric_for_best_model \"f1\" \\\n",
        "\t    --save_total_limit 1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Hb6XFUbZePae",
        "outputId": "9b1cf227-51ce-4b08-e5b9-225f37c16576"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "12/11/2021 09:50:09 - WARNING - __main__ -   Process rank: -1, device: cuda:0, n_gpu: 1distributed training: False, 16-bits training: False\n",
            "12/11/2021 09:50:09 - INFO - __main__ -   Training/evaluation parameters TrainingArguments(\n",
            "_n_gpu=1,\n",
            "adafactor=False,\n",
            "adam_beta1=0.9,\n",
            "adam_beta2=0.999,\n",
            "adam_epsilon=1e-08,\n",
            "bf16=False,\n",
            "bf16_full_eval=False,\n",
            "dataloader_drop_last=False,\n",
            "dataloader_num_workers=0,\n",
            "dataloader_pin_memory=True,\n",
            "ddp_find_unused_parameters=None,\n",
            "debug=[],\n",
            "deepspeed=None,\n",
            "disable_tqdm=False,\n",
            "do_eval=True,\n",
            "do_predict=True,\n",
            "do_train=False,\n",
            "eval_accumulation_steps=None,\n",
            "eval_steps=None,\n",
            "evaluation_strategy=IntervalStrategy.EPOCH,\n",
            "fp16=False,\n",
            "fp16_backend=auto,\n",
            "fp16_full_eval=False,\n",
            "fp16_opt_level=O1,\n",
            "gradient_accumulation_steps=1,\n",
            "gradient_checkpointing=False,\n",
            "greater_is_better=True,\n",
            "group_by_length=False,\n",
            "half_precision_backend=auto,\n",
            "hub_model_id=None,\n",
            "hub_strategy=HubStrategy.EVERY_SAVE,\n",
            "hub_token=<HUB_TOKEN>,\n",
            "ignore_data_skip=False,\n",
            "label_names=None,\n",
            "label_smoothing_factor=0.0,\n",
            "learning_rate=2e-05,\n",
            "length_column_name=length,\n",
            "load_best_model_at_end=True,\n",
            "local_rank=-1,\n",
            "log_level=-1,\n",
            "log_level_replica=-1,\n",
            "log_on_each_node=True,\n",
            "logging_dir=models/ZeroShot/0/eval-dev/runs/Dec11_09-50-09_91e85175b0fd,\n",
            "logging_first_step=False,\n",
            "logging_nan_inf_filter=True,\n",
            "logging_steps=500,\n",
            "logging_strategy=IntervalStrategy.STEPS,\n",
            "lr_scheduler_type=SchedulerType.LINEAR,\n",
            "max_grad_norm=1.0,\n",
            "max_steps=-1,\n",
            "metric_for_best_model=f1,\n",
            "mp_parameters=,\n",
            "no_cuda=False,\n",
            "num_train_epochs=9.0,\n",
            "output_dir=models/ZeroShot/0/eval-dev/,\n",
            "overwrite_output_dir=False,\n",
            "past_index=-1,\n",
            "per_device_eval_batch_size=8,\n",
            "per_device_train_batch_size=32,\n",
            "prediction_loss_only=False,\n",
            "push_to_hub=False,\n",
            "push_to_hub_model_id=None,\n",
            "push_to_hub_organization=None,\n",
            "push_to_hub_token=<PUSH_TO_HUB_TOKEN>,\n",
            "remove_unused_columns=True,\n",
            "report_to=['tensorboard'],\n",
            "resume_from_checkpoint=None,\n",
            "run_name=models/ZeroShot/0/eval-dev/,\n",
            "save_on_each_node=False,\n",
            "save_steps=500,\n",
            "save_strategy=IntervalStrategy.EPOCH,\n",
            "save_total_limit=1,\n",
            "seed=0,\n",
            "sharded_ddp=[],\n",
            "skip_memory_metrics=True,\n",
            "tf32=None,\n",
            "tpu_metrics_debug=False,\n",
            "tpu_num_cores=None,\n",
            "use_legacy_prediction_loop=False,\n",
            "warmup_ratio=0.0,\n",
            "warmup_steps=0,\n",
            "weight_decay=0.0,\n",
            "xpu_backend=None,\n",
            ")\n",
            "12/11/2021 09:50:09 - INFO - __main__ -   load a local file for train: Data/ZeroShot/train.csv\n",
            "12/11/2021 09:50:09 - INFO - __main__ -   load a local file for validation: Data/ZeroShot/dev.csv\n",
            "12/11/2021 09:50:09 - INFO - __main__ -   load a local file for test: Data/ZeroShot/dev.csv\n",
            "12/11/2021 09:50:10 - WARNING - datasets.builder -   Using custom data configuration default-673bcbe64ee9a74c\n",
            "12/11/2021 09:50:10 - WARNING - datasets.builder -   Reusing dataset csv (/root/.cache/huggingface/datasets/csv/default-673bcbe64ee9a74c/0.0.0/bf68a4c4aefa545d0712b2fcbb1b327f905bbe2f6425fbc5e8c25234acb9e14a)\n",
            "100% 3/3 [00:00<00:00, 809.40it/s]\n",
            "[INFO|configuration_utils.py:602] 2021-12-11 09:50:10,160 >> loading configuration file /content/models/ZeroShot/1/config.json\n",
            "[INFO|configuration_utils.py:641] 2021-12-11 09:50:10,162 >> Model config RobertaConfig {\n",
            "  \"_name_or_path\": \"/content/models/ZeroShot/1\",\n",
            "  \"architectures\": [\n",
            "    \"RobertaForSequenceClassification\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"problem_type\": \"single_label_classification\",\n",
            "  \"torch_dtype\": \"float32\",\n",
            "  \"transformers_version\": \"4.14.0.dev0\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "[INFO|tokenization_utils_base.py:1671] 2021-12-11 09:50:10,170 >> Didn't find file /content/models/ZeroShot/1/added_tokens.json. We won't load it.\n",
            "[INFO|tokenization_utils_base.py:1740] 2021-12-11 09:50:10,170 >> loading file /content/models/ZeroShot/1/vocab.json\n",
            "[INFO|tokenization_utils_base.py:1740] 2021-12-11 09:50:10,170 >> loading file /content/models/ZeroShot/1/merges.txt\n",
            "[INFO|tokenization_utils_base.py:1740] 2021-12-11 09:50:10,170 >> loading file /content/models/ZeroShot/1/tokenizer.json\n",
            "[INFO|tokenization_utils_base.py:1740] 2021-12-11 09:50:10,170 >> loading file None\n",
            "[INFO|tokenization_utils_base.py:1740] 2021-12-11 09:50:10,170 >> loading file /content/models/ZeroShot/1/special_tokens_map.json\n",
            "[INFO|tokenization_utils_base.py:1740] 2021-12-11 09:50:10,170 >> loading file /content/models/ZeroShot/1/tokenizer_config.json\n",
            "[INFO|modeling_utils.py:1350] 2021-12-11 09:50:10,282 >> loading weights file /content/models/ZeroShot/1/pytorch_model.bin\n",
            "[INFO|modeling_utils.py:1619] 2021-12-11 09:50:11,903 >> All model checkpoint weights were used when initializing RobertaForSequenceClassification.\n",
            "\n",
            "[INFO|modeling_utils.py:1628] 2021-12-11 09:50:11,903 >> All the weights of RobertaForSequenceClassification were initialized from the model checkpoint at /content/models/ZeroShot/1.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use RobertaForSequenceClassification for predictions without further training.\n",
            "100% 5/5 [00:01<00:00,  3.71ba/s]\n",
            "100% 1/1 [00:00<00:00,  4.44ba/s]\n",
            "100% 1/1 [00:00<00:00,  4.66ba/s]\n",
            "12/11/2021 09:50:17 - INFO - __main__ -   *** Evaluate ***\n",
            "[INFO|trainer.py:549] 2021-12-11 09:50:17,100 >> The following columns in the evaluation set  don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: sentence1.\n",
            "[INFO|trainer.py:2281] 2021-12-11 09:50:17,102 >> ***** Running Evaluation *****\n",
            "[INFO|trainer.py:2283] 2021-12-11 09:50:17,102 >>   Num examples = 739\n",
            "[INFO|trainer.py:2286] 2021-12-11 09:50:17,102 >>   Batch size = 8\n",
            "100% 93/93 [00:11<00:00,  8.04it/s]\n",
            "***** eval metrics *****\n",
            "  eval_accuracy           =     0.6915\n",
            "  eval_f1                 =     0.6877\n",
            "  eval_loss               =      2.049\n",
            "  eval_runtime            = 0:00:11.71\n",
            "  eval_samples            =        739\n",
            "  eval_samples_per_second =     63.071\n",
            "  eval_steps_per_second   =      7.937\n",
            "12/11/2021 09:50:28 - INFO - __main__ -   *** Test ***\n",
            "/content/AStitchInLanguageModels/Dataset/Task2/Utils/run_glue_f1_macro.py:514: FutureWarning: remove_columns_ is deprecated and will be removed in the next major version of datasets. Use Dataset.remove_columns instead.\n",
            "  test_dataset.remove_columns_(\"label\")\n",
            "[INFO|trainer.py:549] 2021-12-11 09:50:28,825 >> The following columns in the test set  don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: sentence1.\n",
            "[INFO|trainer.py:2281] 2021-12-11 09:50:28,826 >> ***** Running Prediction *****\n",
            "[INFO|trainer.py:2283] 2021-12-11 09:50:28,826 >>   Num examples = 739\n",
            "[INFO|trainer.py:2286] 2021-12-11 09:50:28,826 >>   Batch size = 8\n",
            " 99% 92/93 [00:11<00:00,  7.89it/s]12/11/2021 09:50:40 - INFO - __main__ -   ***** Test results None *****\n",
            "100% 93/93 [00:11<00:00,  8.00it/s]\n"
          ]
        }
      ],
      "source": [
        "!python /content/AStitchInLanguageModels/Dataset/Task2/Utils/run_glue_f1_macro.py \\\n",
        "    \t--model_name_or_path '/content/models/ZeroShot/1' \\\n",
        "    \t--do_predict \\\n",
        "    \t--max_seq_length 128 \\\n",
        "    \t--per_device_train_batch_size 32 \\\n",
        "    \t--learning_rate 2e-5 \\\n",
        "    \t--num_train_epochs 9 \\\n",
        "    \t--evaluation_strategy \"epoch\" \\\n",
        "    \t--output_dir models/ZeroShot/0/eval-dev/ \\\n",
        "    \t--seed 0 \\\n",
        "    \t--train_file      Data/ZeroShot/train.csv \\\n",
        "    \t--validation_file Data/ZeroShot/dev.csv \\\n",
        "      --test_file Data/ZeroShot/dev.csv \\\n",
        "\t    --evaluation_strategy \"epoch\" \\\n",
        "\t    --save_strategy \"epoch\"  \\\n",
        "\t    --load_best_model_at_end \\\n",
        "\t    --metric_for_best_model \"f1\" \\\n",
        "\t    --save_total_limit 1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bqkN_56-hmug"
      },
      "outputs": [],
      "source": [
        "params = {\n",
        "    'submission_format_file' : '/content/SemEval_2022_Task2-idiomaticity/SubTaskA/Data/dev_submission_format.csv' ,\n",
        "    'input_file'             : '/content/SemEval_2022_Task2-idiomaticity/SubTaskA/Data/dev.csv'                   ,\n",
        "    'prediction_format_file' : '/content/models/ZeroShot/0/eval-dev/test_results_None.txt'                        ,\n",
        "    }\n",
        "params[ 'setting' ] = 'zero_shot'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Mz_6R1aD559C"
      },
      "outputs": [],
      "source": [
        " updated_data = insert_to_submission_file( **params )"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mS9ymlTY57lY"
      },
      "outputs": [],
      "source": [
        "!mkdir -p outputs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vdaFWbcn5857",
        "outputId": "8402ee36-7d36-4b64-8828-3f7341c9e425"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Wrote outputs/zero_shot_dev_formated.csv\n"
          ]
        }
      ],
      "source": [
        "write_csv( updated_data, 'outputs/zero_shot_dev_formated.csv' ) "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 218
        },
        "id": "xtSW1IaF5-Zw",
        "outputId": "3c018c71-1d77-4ae2-920f-146e1a2f9bcd"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.google.colaboratory.module+javascript": "\n      import \"https://ssl.gstatic.com/colaboratory/data_table/a6224c040fa35dcf/data_table.js\";\n\n      window.createDataTable({\n        data: [[{\n            'v': 0,\n            'f': \"0\",\n        },\n\"zero_shot\",\n\"EN\",\n0.7345693014159194],\n [{\n            'v': 1,\n            'f': \"1\",\n        },\n\"zero_shot\",\n\"PT\",\n0.5602510050004903],\n [{\n            'v': 2,\n            'f': \"2\",\n        },\n\"zero_shot\",\n\"EN,PT\",\n0.6877233301208392],\n [{\n            'v': 3,\n            'f': \"3\",\n        },\n\"one_shot\",\n\"EN\",\n[null, null, null]],\n [{\n            'v': 4,\n            'f': \"4\",\n        },\n\"one_shot\",\n\"PT\",\n[null, null, null]],\n [{\n            'v': 5,\n            'f': \"5\",\n        },\n\"one_shot\",\n\"EN,PT\",\n[null, null, null]]],\n        columns: [[\"number\", \"index\"], [\"string\", \"Settings\"], [\"string\", \"Languages\"], [\"string\", \"F1 Score (Macro)\"]],\n        columnOptions: [{\"width\": \"1px\", \"className\": \"index_column\"}],\n        rowsPerPage: 25,\n        helpUrl: \"https://colab.research.google.com/notebooks/data_table.ipynb\",\n        suppressOutputScrolling: true,\n        minimumWidth: undefined,\n      });\n    ",
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Settings</th>\n",
              "      <th>Languages</th>\n",
              "      <th>F1 Score (Macro)</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>zero_shot</td>\n",
              "      <td>EN</td>\n",
              "      <td>0.734569</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>zero_shot</td>\n",
              "      <td>PT</td>\n",
              "      <td>0.560251</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>zero_shot</td>\n",
              "      <td>EN,PT</td>\n",
              "      <td>0.687723</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>one_shot</td>\n",
              "      <td>EN</td>\n",
              "      <td>(None, None, None)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>one_shot</td>\n",
              "      <td>PT</td>\n",
              "      <td>(None, None, None)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>one_shot</td>\n",
              "      <td>EN,PT</td>\n",
              "      <td>(None, None, None)</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    Settings Languages    F1 Score (Macro)\n",
              "0  zero_shot        EN            0.734569\n",
              "1  zero_shot        PT            0.560251\n",
              "2  zero_shot     EN,PT            0.687723\n",
              "3   one_shot        EN  (None, None, None)\n",
              "4   one_shot        PT  (None, None, None)\n",
              "5   one_shot     EN,PT  (None, None, None)"
            ]
          },
          "execution_count": 25,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "import sys\n",
        "sys.path.append( '/content/SemEval_2022_Task2-idiomaticity/SubTaskA/' ) \n",
        "from SubTask1Evaluator import evaluate_submission\n",
        "\n",
        "\n",
        "submission_file = 'outputs/zero_shot_dev_formated.csv'\n",
        "gold_file       = '/content/SemEval_2022_Task2-idiomaticity/SubTaskA/Data/dev_gold.csv'\n",
        "\n",
        "results = evaluate_submission( submission_file, gold_file )\n",
        "%reload_ext google.colab.data_table\n",
        "import pandas as pd\n",
        "df = pd.DataFrame(data=results[1:], columns=results[0])\n",
        "df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Airuohmm6AEL"
      },
      "outputs": [],
      "source": [
        ""
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "name": "zero_shot_simple_roberta.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}