AnalysisUserStudy.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Analysis of our user study"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this notebook we provide the code that was used to obtain the results from the user study shown in our paper."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import json\n",
    "from pandas import DataFrame\n",
    "import math\n",
    "\n",
    "import torch\n",
    "import pickle\n",
    "import matplotlib.pyplot as plt\n",
    "from concept_reader import ConceptExemplar, read_concepts, Concept\n",
    "import torchvision.transforms as transforms\n",
    "#import clip\n",
    "from PIL import Image\n",
    "import numpy\n",
    "from collections import OrderedDict\n",
    "from tqdm import tqdm\n",
    "import json\n",
    "import cv2\n",
    "import pandas as pd\n",
    "from pandas import DataFrame\n",
    "\n",
    "import itertools\n",
    "\n",
    "import nltk\n",
    "from nltk.corpus import wordnet\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "wordnet_lemmatizer = WordNetLemmatizer()\n",
    "\n",
    "\n",
    "import gensim\n",
    "from gensim.models import CoherenceModel, KeyedVectors\n",
    "from gensim.test.utils import common_corpus, common_dictionary\n",
    "from gensim.models.coherencemodel import CoherenceModel\n",
    "\n",
    "device = \"cuda:1\" if torch.cuda.is_available() else \"cpu\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data loading and preparation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load the user study data files, that contain the annotations for the concept examples.\n",
    "We collected 2600 human assessments of visual concepts from students for this experiment. They data contains a meaning annotated for the concept in form of a word and a numerical rating of the fit of the description in the range 0 (not at all) to 3 (very well)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded 2600 ground truth concept ratings.\n"
     ]
    }
   ],
   "source": [
    "# Load the data\n",
    "full_list = []\n",
    "for i in range(10):\n",
    "    with open(f\"data/user_study/annotations{i+1:02d}.json\") as file: # Ignore the syntax highlighting, this is correct.\n",
    "        mylist = json.load(file)\n",
    "        #print(len(mylist))\n",
    "        full_list.extend(mylist)\n",
    "mylist = full_list\n",
    "print(\"Loaded\", len(mylist), \"ground truth concept ratings.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Some post-processing of the results. \n",
    "Within our dataset, we annotated which algorithm provided the words chosen by the user, to study which method produced the best-fitting descriptions. However, sometimes there were singular and plural names of the same words provided, which we fuse in postprocessing.\n",
    "If the corresponding word was chosen, both algorithms obtained a point."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "process = words_lists[2]\n",
    "for word in process:\n",
    "    for item in word[\"word_list\"]:\n",
    "        item[\"score\"] = -1*item[\"score\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Merge singular and plural words, e.g. \"door\", \"doors\". \n",
    "# The lists of algorithms that provided these words will be fused as well.\n",
    "\n",
    "# List of annotations (meanings) for the concepts provided by our annotation modules (OC, JVL, JVL with cropping)\n",
    "word_files = {0: \"data/user_study/oc_words.json\", 1: \"data/user_study/jvl_words_mse_10kwords.json\", 2: \"data/user_study/jvl_crop_words_mse_10kwords.json\"}\n",
    "\n",
    "words_lists = {k: json.load(open(v, \"r\")) for k, v in word_files.items()}\n",
    "\n",
    "def fuse_wordlists(file_name, infile_idx, items):\n",
    "    \"\"\" Return a list of the words from all algorithms. \"\"\"\n",
    "    \n",
    "    num_words = 4 # Max number of words displayed\n",
    "    word_dict = {}\n",
    "    for alg_id, v in words_lists.items():\n",
    "        for item in v:\n",
    "            if item[\"source_file\"] == file_name and item[\"source_id\"] == infile_idx:\n",
    "                num_items = 0\n",
    "                for witem in item[\"word_list\"]:\n",
    "                    if num_items >= num_words:\n",
    "                        break   \n",
    "                    if witem[\"word\"] in word_dict:\n",
    "                        word_dict[witem[\"word\"]].append(alg_id)\n",
    "                    else:\n",
    "                        word_dict[witem[\"word\"]] = [alg_id]\n",
    "                    num_items += 1\n",
    "    own_entries = find_all_own_words(file_name, infile_idx, items)\n",
    "    for e in own_entries:\n",
    "        if e in word_dict:\n",
    "            word_dict[e].append(3)\n",
    "        else:\n",
    "            word_dict[e] = [3]\n",
    "        \n",
    "    unmerged_list =  [(k,v) for k, v in word_dict.items()]\n",
    "    merged_list = [k for (k,v) in unmerged_list]\n",
    "    \n",
    "    for word in merged_list: # Remove plural if it is also in list.\n",
    "        if word + \"s\" in merged_list:\n",
    "            merged_list.remove(word + \"s\")\n",
    "\n",
    "    return unmerged_list, merged_list\n",
    "   \n",
    "def find_all_own_words(source_file, source_id, item_list):\n",
    "    ret_list = []\n",
    "    for item in mylist:\n",
    "        if item[\"source_file\"] == source_file and item[\"source_id\"] == source_id:\n",
    "            if item[\"description_algorithm\"] == [3]: # 3 means user entry\n",
    "                ret_list.append(item[\"description\"])\n",
    "    return ret_list\n",
    "\n",
    "for item in mylist:\n",
    "    if len(item[\"description_algorithm\"]) == 0:\n",
    "        item[\"description_algorithm\"] =  [3]\n",
    "        \n",
    "for item in mylist:\n",
    "    corresponding_words, merged_list = fuse_wordlists(item[\"source_file\"], item[\"source_id\"], mylist)\n",
    "    if item[\"description\"][-1] == \"s\": # Try removing the plural\n",
    "        for word, alg in corresponding_words:\n",
    "            if word == item[\"description\"][:-1]:\n",
    "                #print(word, item[\"description\"])\n",
    "                myset = set(item[\"description_algorithm\"])\n",
    "                myset.update(alg)\n",
    "                item[\"description_algorithm\"] = list(myset)\n",
    "                item[\"description\"] = item[\"description\"][:-1] # Remove plural.\n",
    "    else: # Try mergin with the plural\n",
    "        for word, alg in corresponding_words:\n",
    "            if word == item[\"description\"]+  \"s\":\n",
    "                #print(word, item[\"description\"])\n",
    "                myset = set(item[\"description_algorithm\"])\n",
    "                myset.update(alg)\n",
    "                item[\"description_algorithm\"] = list(myset)\n",
    "    #print(item[\"source_file\"], item[\"source_id\"], item)\n",
    "    try:\n",
    "        item[\"description_idx\"] = merged_list.index(item[\"description\"])\n",
    "    except ValueError:\n",
    "        item[\"description_idx\"] = 0\n",
    "        \n",
    "    item[\"num_word_choices\"] = len(merged_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load data in to a ``pandas.Dataframe``. Also load the Cross-Validation Splits (the splits we initially used were generated randomly. For reproducability, we stored them in an additional file and load them)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load Dataframe and the Cross-Validation Splits (the splits we initially used were generated randomly. For re)\n",
    "data_frame = DataFrame.from_dict(mylist)\n",
    "\n",
    "split_dict = {} # Map (source_file, source_id) -> is_test\n",
    "import csv\n",
    "with open('data/user_study/split.csv', 'r', newline='\\r\\n') as incsv:\n",
    "    reader = csv.reader(incsv)\n",
    "    first_row = True\n",
    "    for row in reader:\n",
    "        if first_row:\n",
    "            first_row = False\n",
    "            continue\n",
    "        split_dict[(row[0], int(row[1]))] = int(row[2])\n",
    "\n",
    "#print(split_dict)\n",
    "\n",
    "is_test_list = []\n",
    "for i in range(len(data_frame)):\n",
    "    is_test_list.append(split_dict[(data_frame[\"source_file\"].iloc[i], data_frame[\"source_id\"].iloc[i])])\n",
    "x = pd.Series(is_test_list, name=\"fold\")\n",
    "data_frame[\"fold\"] = x\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source_file</th>\n",
       "      <th>source_id</th>\n",
       "      <th>rating</th>\n",
       "      <th>rating_num</th>\n",
       "      <th>description</th>\n",
       "      <th>description_algorithm</th>\n",
       "      <th>description_idx</th>\n",
       "      <th>num_word_choices</th>\n",
       "      <th>fold</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>places365_kMeans</td>\n",
       "      <td>21</td>\n",
       "      <td>very well</td>\n",
       "      <td>3</td>\n",
       "      <td>tower</td>\n",
       "      <td>[0, 1, 2]</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>awa_mixin10</td>\n",
       "      <td>7</td>\n",
       "      <td>not at all</td>\n",
       "      <td>0</td>\n",
       "      <td>a</td>\n",
       "      <td>[3]</td>\n",
       "      <td>11</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>awa_kMeans</td>\n",
       "      <td>10</td>\n",
       "      <td>mostly well</td>\n",
       "      <td>2</td>\n",
       "      <td>background</td>\n",
       "      <td>[3]</td>\n",
       "      <td>10</td>\n",
       "      <td>13</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>places365_mixin5</td>\n",
       "      <td>17</td>\n",
       "      <td>to some extend</td>\n",
       "      <td>1</td>\n",
       "      <td>monitor</td>\n",
       "      <td>[3]</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>places365_kMeans</td>\n",
       "      <td>40</td>\n",
       "      <td>very well</td>\n",
       "      <td>3</td>\n",
       "      <td>water</td>\n",
       "      <td>[0]</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2595</th>\n",
       "      <td>places365_mixin10</td>\n",
       "      <td>18</td>\n",
       "      <td>not at all</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>[3]</td>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2596</th>\n",
       "      <td>awa_mixin5</td>\n",
       "      <td>16</td>\n",
       "      <td>to some extend</td>\n",
       "      <td>1</td>\n",
       "      <td>nature</td>\n",
       "      <td>[3]</td>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2597</th>\n",
       "      <td>awa_Shap</td>\n",
       "      <td>20</td>\n",
       "      <td>mostly well</td>\n",
       "      <td>2</td>\n",
       "      <td>biodiversity</td>\n",
       "      <td>[2]</td>\n",
       "      <td>11</td>\n",
       "      <td>16</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2598</th>\n",
       "      <td>places365_Shap</td>\n",
       "      <td>0</td>\n",
       "      <td>mostly well</td>\n",
       "      <td>2</td>\n",
       "      <td>dinner_table</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>6</td>\n",
       "      <td>9</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2599</th>\n",
       "      <td>places365_kMeans</td>\n",
       "      <td>5</td>\n",
       "      <td>not at all</td>\n",
       "      <td>0</td>\n",
       "      <td>bed</td>\n",
       "      <td>[0, 1, 2]</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2600 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            source_file  source_id          rating  rating_num   description  \\\n",
       "0      places365_kMeans         21       very well           3         tower   \n",
       "1           awa_mixin10          7      not at all           0             a   \n",
       "2            awa_kMeans         10     mostly well           2    background   \n",
       "3      places365_mixin5         17  to some extend           1       monitor   \n",
       "4      places365_kMeans         40       very well           3         water   \n",
       "...                 ...        ...             ...         ...           ...   \n",
       "2595  places365_mixin10         18      not at all           0             0   \n",
       "2596         awa_mixin5         16  to some extend           1        nature   \n",
       "2597           awa_Shap         20     mostly well           2  biodiversity   \n",
       "2598     places365_Shap          0     mostly well           2  dinner_table   \n",
       "2599   places365_kMeans          5      not at all           0           bed   \n",
       "\n",
       "     description_algorithm  description_idx  num_word_choices  fold  \n",
       "0                [0, 1, 2]                2                 7     0  \n",
       "1                      [3]               11                13     2  \n",
       "2                      [3]               10                13     7  \n",
       "3                      [3]                9                10     3  \n",
       "4                      [0]                0                 8     5  \n",
       "...                    ...              ...               ...   ...  \n",
       "2595                   [3]               10                11     4  \n",
       "2596                   [3]               11                12     0  \n",
       "2597                   [2]               11                16     7  \n",
       "2598                [1, 2]                6                 9     5  \n",
       "2599             [0, 1, 2]                0                 9     3  \n",
       "\n",
       "[2600 rows x 9 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_frame"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Description of the columns:\n",
    "\n",
    "* ``source_file``: Source of the concept of the form ``<dataset>_<algorithm>``.\n",
    "* ``source_id``: Running number in the concept file. source_file and source_id together form a unique identifier for a specific concept. In total, there are 260 concepts.\n",
    "* ``rating``: Rating assigned by the annotator in text form\n",
    "* ``rating_num``: Rating converted to a numerical score. \n",
    "* ``description``: Single-word description assigned to the concept by the user.\n",
    "* ``description_algorithm``: Which algorithm provided the description. The user was offered a choice of words by the annotation modules to choose from. If they picked one of these choices, this field indicates which algorithm came up with this word. 0 means the word was entered manually.\n",
    "* ``decription_idx``: Numeric index of the word out of the choices. Same words have the same index (in the same concept).\n",
    "* ``num_word_choice``: Number of words that were given as a choice for the rater + numer of freely chosen words (range of ``description_idx``).\n",
    "* ``fold`` cross-validation fold number (0-9)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Example: Show all ratings for a specific conept."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source_file</th>\n",
       "      <th>source_id</th>\n",
       "      <th>rating</th>\n",
       "      <th>rating_num</th>\n",
       "      <th>description</th>\n",
       "      <th>description_algorithm</th>\n",
       "      <th>description_idx</th>\n",
       "      <th>num_word_choices</th>\n",
       "      <th>fold</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>awa_Shap</td>\n",
       "      <td>8</td>\n",
       "      <td>to some extend</td>\n",
       "      <td>1</td>\n",
       "      <td>mane</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>4</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>404</th>\n",
       "      <td>awa_Shap</td>\n",
       "      <td>8</td>\n",
       "      <td>not at all</td>\n",
       "      <td>0</td>\n",
       "      <td>wall</td>\n",
       "      <td>[0]</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>702</th>\n",
       "      <td>awa_Shap</td>\n",
       "      <td>8</td>\n",
       "      <td>to some extend</td>\n",
       "      <td>1</td>\n",
       "      <td>fur</td>\n",
       "      <td>[2]</td>\n",
       "      <td>8</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>929</th>\n",
       "      <td>awa_Shap</td>\n",
       "      <td>8</td>\n",
       "      <td>mostly well</td>\n",
       "      <td>2</td>\n",
       "      <td>fur</td>\n",
       "      <td>[2]</td>\n",
       "      <td>8</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1162</th>\n",
       "      <td>awa_Shap</td>\n",
       "      <td>8</td>\n",
       "      <td>mostly well</td>\n",
       "      <td>2</td>\n",
       "      <td>fur</td>\n",
       "      <td>[2]</td>\n",
       "      <td>8</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1465</th>\n",
       "      <td>awa_Shap</td>\n",
       "      <td>8</td>\n",
       "      <td>mostly well</td>\n",
       "      <td>2</td>\n",
       "      <td>mane</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>4</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1679</th>\n",
       "      <td>awa_Shap</td>\n",
       "      <td>8</td>\n",
       "      <td>mostly well</td>\n",
       "      <td>2</td>\n",
       "      <td>mane</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>4</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1876</th>\n",
       "      <td>awa_Shap</td>\n",
       "      <td>8</td>\n",
       "      <td>not at all</td>\n",
       "      <td>0</td>\n",
       "      <td>hairy</td>\n",
       "      <td>[1]</td>\n",
       "      <td>7</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2196</th>\n",
       "      <td>awa_Shap</td>\n",
       "      <td>8</td>\n",
       "      <td>mostly well</td>\n",
       "      <td>2</td>\n",
       "      <td>fur</td>\n",
       "      <td>[2]</td>\n",
       "      <td>8</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2485</th>\n",
       "      <td>awa_Shap</td>\n",
       "      <td>8</td>\n",
       "      <td>mostly well</td>\n",
       "      <td>2</td>\n",
       "      <td>animals</td>\n",
       "      <td>[3]</td>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     source_file  source_id          rating  rating_num description  \\\n",
       "46      awa_Shap          8  to some extend           1        mane   \n",
       "404     awa_Shap          8      not at all           0        wall   \n",
       "702     awa_Shap          8  to some extend           1         fur   \n",
       "929     awa_Shap          8     mostly well           2         fur   \n",
       "1162    awa_Shap          8     mostly well           2         fur   \n",
       "1465    awa_Shap          8     mostly well           2        mane   \n",
       "1679    awa_Shap          8     mostly well           2        mane   \n",
       "1876    awa_Shap          8      not at all           0       hairy   \n",
       "2196    awa_Shap          8     mostly well           2         fur   \n",
       "2485    awa_Shap          8     mostly well           2     animals   \n",
       "\n",
       "     description_algorithm  description_idx  num_word_choices  fold  \n",
       "46                  [1, 2]                4                11     0  \n",
       "404                    [0]                0                11     0  \n",
       "702                    [2]                8                11     0  \n",
       "929                    [2]                8                11     0  \n",
       "1162                   [2]                8                11     0  \n",
       "1465                [1, 2]                4                11     0  \n",
       "1679                [1, 2]                4                11     0  \n",
       "1876                   [1]                7                11     0  \n",
       "2196                   [2]                8                11     0  \n",
       "2485                   [3]               10                11     0  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_frame[(data_frame[\"source_id\"]==8) & (data_frame[\"source_file\"]==\"awa_Shap\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Manual                  0.130337\n",
       "OC                      0.525393\n",
       "JVL                     0.365843\n",
       "JVL (cropped images)    0.476404\n",
       "dtype: float64"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Evaluate the description algorithms\n",
    "data_frame[\"description_algorithm\"] = data_frame[\"description_algorithm\"].astype(str)\n",
    "data_frame_filtered = data_frame[data_frame[\"rating_num\"] >= 1]\n",
    "all_contains = pd.concat({\"Manual\": data_frame_filtered[\"description_algorithm\"].str.contains(\"3\"), \"OC\": data_frame_filtered[\"description_algorithm\"].str.contains(\"0\"), \"JVL\": data_frame_filtered[\"description_algorithm\"].str.contains(\"1\"), \"JVL (cropped images)\": data_frame_filtered[\"description_algorithm\"].str.contains(\"2\")}, axis=1)\n",
    "agg = all_contains.sum(axis=0)\n",
    "agg/all_contains.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'words chosen')"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAEICAYAAAC9E5gJAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAXMUlEQVR4nO3de5hddX3v8fdHolzEC0ikgSCJGtGAiBpSvCEtWtKChnqgxnqJFUUtrR6P59FgfdSeNi3Hes6j1mIP3ojKCUaPSipVxChegRiEAiEgkSBEIgwqAl4ol+/5Y/1SVoeZZJw9zEzk/Xqe/ey1fuu31vruSWZ/1m+tvdekqpAkPbA9aKoLkCRNPcNAkmQYSJIMA0kShoEkCcNAkoRhIJHkvCSvnsDtnZ7kbydqe9JkMAwkSYaBHjjS8f+8NAJ/MTQtJfmzJP/Sm9+YZFVv/vokh7TpZyb5bpKft+dn9vqdl2R5km8DvwQem+T5Sa5s/T8ApNf/8Um+3pbdnORT26jx2Um+k+SWVs8re4v3SHJ2ktuSXJjkcb31tlXvK5Nc09bblOSlvWWvSrIhyc+SnJNk/96ySvK6JFe35f+UJEhjVVU+fEy7B/BY4Ba6A5ZZwA+BH/WW/awt27NNvxyYAbykzT+q9T0PuA44sC2fCdwKHAc8GHgTcBfw6tZ/JfBXbdu7AM8epb7HALe1/T0YeBRwSFt2OvBTYGHb5xnAmW3ZqPUCD221HdD6zgIObNPHAhuBJ7X13g58p1dPAV8AHtlqGwIWTfW/o48d5+HIQNNSVV1D92Z7CPBc4BzgR0me2Oa/WVX3AEcDV1fVJ6rqrqpaCVwJvKC3udOran1V3QX8IXBFVX2mqu4E3gv8uNf3TmB/YJ+q+nVVfWuUEl8KfKWqVlbVnVX1k6q6pLf8s1W1tu3zjPY6GEO99wAHJdm1qrZU1frW/lrg76tqQ9vm3wGH9EcHwClVdUtVXQd8rbdPabsMA01nXweOAA5v0+fRBcFz2zzAPnSjhr4fAvv25q/vTe/Tn6+qGrb8LXSnjdYmWZ/kVaPUth/wg23U3g+YXwK7b6/eqvoF8GLgdcCWdprpia3P/sD72impW+hGHhn2Okfbp7RdhoGms61h8Jw2/XXuGwY30L1R9j0G+FFvvn9r3i10b+RAd1G5P19VP66q11TVPnRH46cmefwItV0PPG6E9u3ZZr1VdU5VPZ/uFNGVwId6+3ttVT2y99i1qr4zjhqk+zAMNJ19Hfg9YNeq2gx8E1hEd3794tbnX4EnJPnTJDOSvBiYT3f+fCRnAwcmeVGSGcAbgN/ZujDJ8Ulmt9mf0QXJ3SNs5wzgeUn+pO33UVsvaG/HqPUm2TvJC5M8FLgDuL23738GTk5yYKvzEUmOH8P+pDExDDRtVdX36d4Qv9nmbwWuAb5dVXe3tp8AxwBvBn5Cd5rnmKq6eZRt3gwcD5zS+s8Dvt3rcihwYZLbgdXAG6tq0wjbuQ74o7bfnwKXAE8Zw2vaVr0Pau03tG0+F/jztt7ngP8JnJnkVuByuusf0oRId8pUkvRA5shAkmQYSJIMA0kShoEkie5r7dPaXnvtVXPmzJnqMiRph3LRRRfdXFUzx9p/2ofBnDlzWLdu3VSXIUk7lCTDv+m+Tds9TZTko0luSnJ5r+0f2l0fL03yuSSP7C07ud1h8qokR/Xan57ksrbs/d5RUZKmj7FcMzid7luffecCB1XVwcD3gZMBkswHltDdIXIR3Vf5d2rrfBA4ke5LPvNG2KYkaYpsNwyq6ht034bst3253TkR4AJg69f3F9PdqveO9q3NjcDCJLOAh1fV+e3GYB+nuyWvJGkamIhPE70K+GKb3pf/fAfIza1t3zY9vH1ESU5Msi7JuqGhoQkoUZK0LQOFQZK/ovvDIGdsbRqhW22jfURVdVpVLaiqBTNnjvliuCRpnMb9aaIkS+luuHVk3XuDo830bgdMd/rohtY+e4R2SdI0MK6RQZJFwFuBF1bVL3uLVgNLkuycZC7dheK1VbUFuC3JYe1TRK8AzhqwdknSBNnuyCDJSro/MLJXks3AO+k+PbQzcG77hOgFVfW6qlrf/mj5FXSnj07aeqth4PV0n0zale4awxeRJE0L0/4W1gsWLCi/dCZJv5kkF1XVgrH2n/bfQNbUmbPs7KkuYUpde8rRU12CNGm8UZ0kyTCQJBkGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJ+GcvJU1T/tnVyf2zq44MJEmGgSTJMJAkYRhIkjAMJEkYBpIkxhAGST6a5KYkl/fa9kxybpKr2/MevWUnJ9mY5KokR/Xan57ksrbs/Uky8S9HkjQeYxkZnA4sGta2DFhTVfOANW2eJPOBJcCBbZ1Tk+zU1vkgcCIwrz2Gb1OSNEW2GwZV9Q3gp8OaFwMr2vQK4Nhe+5lVdUdVbQI2AguTzAIeXlXnV1UBH++tI0maYuO9ZrB3VW0BaM+Pbu37Atf3+m1ubfu26eHtkqRpYKIvII90HaC20T7yRpITk6xLsm5oaGjCipMkjWy8YXBjO/VDe76ptW8G9uv1mw3c0Npnj9A+oqo6raoWVNWCmTNnjrNESdJYjTcMVgNL2/RS4Kxe+5IkOyeZS3eheG07lXRbksPap4he0VtHkjTFtnvX0iQrgSOAvZJsBt4JnAKsSnICcB1wPEBVrU+yCrgCuAs4qarubpt6Pd0nk3YFvtgekqRpYLthUFUvGWXRkaP0Xw4sH6F9HXDQb1SdJGlS+A1kSZJhIEkyDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJDBgGSd6UZH2Sy5OsTLJLkj2TnJvk6va8R6//yUk2JrkqyVGDly9Jmggzxrtikn2BNwDzq+pXSVYBS4D5wJqqOiXJMmAZ8NYk89vyA4F9gK8keUJV3T3wq5CmoTnLzp7qEqbUtaccPdUl6Dcw6GmiGcCuSWYAuwE3AIuBFW35CuDYNr0YOLOq7qiqTcBGYOGA+5ckTYBxh0FV/Qh4D3AdsAX4eVV9Gdi7qra0PluAR7dV9gWu721ic2u7jyQnJlmXZN3Q0NB4S5QkjdG4w6BdC1gMzKU77fPQJC/b1iojtNVIHavqtKpaUFULZs6cOd4SJUljNMhpoucBm6pqqKruBD4LPBO4McksgPZ8U+u/Gdivt/5sutNKkqQpNkgYXAcclmS3JAGOBDYAq4Glrc9S4Kw2vRpYkmTnJHOBecDaAfYvSZog4/40UVVdmOQzwPeAu4CLgdOA3YFVSU6gC4zjW//17RNHV7T+J/lJIkmaHsYdBgBV9U7gncOa76AbJYzUfzmwfJB9SpImnt9AliQZBpIkw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSSJAcMgySOTfCbJlUk2JHlGkj2TnJvk6va8R6//yUk2JrkqyVGDly9JmgiDjgzeB3ypqp4IPAXYACwD1lTVPGBNmyfJfGAJcCCwCDg1yU4D7l+SNAHGHQZJHg4cDnwEoKr+vapuARYDK1q3FcCxbXoxcGZV3VFVm4CNwMLx7l+SNHEGGRk8FhgCPpbk4iQfTvJQYO+q2gLQnh/d+u8LXN9bf3NrkyRNsUHCYAbwNOCDVfVU4Be0U0KjyAhtNWLH5MQk65KsGxoaGqBESdJYDBIGm4HNVXVhm/8MXTjcmGQWQHu+qdd/v976s4EbRtpwVZ1WVQuqasHMmTMHKFGSNBbjDoOq+jFwfZIDWtORwBXAamBpa1sKnNWmVwNLkuycZC4wD1g73v1LkibOjAHX/0vgjCQPAa4B/owuYFYlOQG4DjgeoKrWJ1lFFxh3ASdV1d0D7l+SNAEGCoOqugRYMMKiI0fpvxxYPsg+JUkTz28gS5IMA0mSYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJYgLCIMlOSS5O8oU2v2eSc5Nc3Z736PU9OcnGJFclOWrQfUuSJsZEjAzeCGzozS8D1lTVPGBNmyfJfGAJcCCwCDg1yU4TsH9J0oAGCoMks4GjgQ/3mhcDK9r0CuDYXvuZVXVHVW0CNgILB9m/JGliDDoyeC/wFuCeXtveVbUFoD0/urXvC1zf67e5td1HkhOTrEuybmhoaMASJUnbM+4wSHIMcFNVXTTWVUZoq5E6VtVpVbWgqhbMnDlzvCVKksZoxgDrPgt4YZI/AnYBHp7kk8CNSWZV1ZYks4CbWv/NwH699WcDNwywf0nSBBn3yKCqTq6q2VU1h+7C8Fer6mXAamBp67YUOKtNrwaWJNk5yVxgHrB23JVLkibMICOD0ZwCrEpyAnAdcDxAVa1Psgq4ArgLOKmq7r4f9i9J+g1NSBhU1XnAeW36J8CRo/RbDiyfiH1KkiaO30CWJBkGkiTDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRx/9zCetqYs+zsqS5hSl17ytFTXYKkHYQjA0mSYSBJMgwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkBgiDJPsl+VqSDUnWJ3lja98zyblJrm7Pe/TWOTnJxiRXJTlqIl6AJGlwg4wM7gLeXFVPAg4DTkoyH1gGrKmqecCaNk9btgQ4EFgEnJpkp0GKlyRNjHGHQVVtqarvtenbgA3AvsBiYEXrtgI4tk0vBs6sqjuqahOwEVg43v1LkibOhFwzSDIHeCpwIbB3VW2BLjCAR7du+wLX91bb3NpG2t6JSdYlWTc0NDQRJUqStmHgMEiyO/D/gP9aVbduq+sIbTVSx6o6raoWVNWCmTNnDlqiJGk7BgqDJA+mC4IzquqzrfnGJLPa8lnATa19M7Bfb/XZwA2D7F+SNDEG+TRRgI8AG6rqf/cWrQaWtumlwFm99iVJdk4yF5gHrB3v/iVJE2eQv4H8LODlwGVJLmltbwNOAVYlOQG4DjgeoKrWJ1kFXEH3SaSTquruAfYvSZog4w6DqvoWI18HADhylHWWA8vHu09J0v3DbyBLkgwDSZJhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEliCsIgyaIkVyXZmGTZZO9fknRfkxoGSXYC/gn4Q2A+8JIk8yezBknSfU32yGAhsLGqrqmqfwfOBBZPcg2SpGFSVZO3s+Q4YFFVvbrNvxz43ar6i2H9TgRObLMHAFdNWpETay/g5qkuYgfmz28w/vwGs6P//Pavqplj7Tzj/qxkBBmh7T5pVFWnAafd/+Xcv5Ksq6oFU13Hjsqf32D8+Q3mgfbzm+zTRJuB/Xrzs4EbJrkGSdIwkx0G3wXmJZmb5CHAEmD1JNcgSRpmUk8TVdVdSf4COAfYCfhoVa2fzBom2Q5/qmuK+fMbjD+/wTygfn6TegFZkjQ9+Q1kSZJhIEkyDEaVpJJ8ojc/I8lQki9MYg2vTPKBydrfVEkyO8lZSa5O8oMk72sfMCDJwiTfaLcwuTLJh5PsNtU1TydJbk+yKckBw9rfm+QtSY6YrP+3E1VL63/4/VfpYEZ7HUkWJHn/VNQ0miRfSbLH9voZBqP7BXBQkl3b/POBH01hPb+VkgT4LPD5qpoHPAHYHVieZG/g08Bbq+oA4EnAl4CHTVW909iZdJ/OAyDJg4DjgE/taLUk2RM4rKq+Mcb+k/19qVFV1bqqesNU1zHMJ4A/314nw2Dbvggc3aZfAqzcuqAdsX4nycXt+YDW/sokn03ypXak++7eOrf3po9LcnqbfkGSC9u2vtLeBB8ofh/4dVV9DKCq7gbeBLwKeDOwoqrOb8uqqj5TVTdOWbXT10p6b8DA4cC1VfXDHbCW4+hCH4Akh7bfsX9LsjbJw9rv2aeT/Avw5SR7Jvl8kkuTXJDk4Lbuu5J8IslX2+/ja1r7EW3E+bkkVyT55xZaJPmDJOcn+V7bx+6tfVEbnX4LeNFIhfdHDG3fK5J8Ocm1SV6U5N1JLmvvDw9u/d6R5LtJLk9yWjtA2vq6L221/EOSy1v7Tm3+u235a1v7rPaaLmnbek4razXd+9c2GQbbdiawJMkuwMHAhb1lVwKHV9VTgXcAf9dbdgjwYuDJwIuT9L9oN5Jv0R0JPbXt8y0TU/4O4UDgon5DVd0KXAc8fvgyjayqLgXuSfKU1rSE3sHLDlbLs2j/7u104aeAN1bVU4DnAb9q/Z4BLK2q3wf+Gri4qg4G3gZ8vLe9g+kO6p4BvCPJPq19Id0Bx5OBxwEvSrIX8HbgeVX1NGAd8N/ae8CHgBcAzwF+Z4yv5XFt34uBTwJfq6ont9ew9UDzA1V1aFUdBOwKHNPaPwa8rqqeAdzd2+YJwM+r6lDgUOA1SeYCfwqcU1WHAE8BLgGoqp8BOyd51LYKnTbDq+moqi5NMocuVf912OJHACuSzKO7pcaDe8vWVNXPAZJcAewPXL+NXc0GPpVkFvAQYNPEvIIdQhjhliStfaTbl2h0K+kOXtbTvfm8YwetZRYw1KYPALZU1XfhPw4UaAfP51bVT1u/ZwP/pfX5apJHJXlEW3ZWVf0K+FWSr9GFwC3A2qq6pm1vZdvGr+nuqPztto+HAOcDTwQ2VdXVrf8nuff+advyxaq6M8lldN+t2jriuQyY06Z/L8lbgN2APYH1Sb4JPKyqvtP6/F/uDYk/AA5Od6836N6L5tF9qfejbcTx+aq6pFfHTcA+wE9GK9SRwfatBt7DfY9s/oYu5Q+iO1rYpbfsjt703dwbuv03vX7/f6Q7Ongy8Nphy37brQf+0/1fkjyc7rYlG4GnT0VRO6iVwJ/QHT1fWlU37aC1/Ip7fwdGO1iA7roevX7D1bDnsbSHLmQOaY/5VXXCKP3H4g6AqroHuLPu/WLXPcCMNuI4FTiu/f5/iO61b+tAKMBf9mqcW1VfbtdYDqe7tvmJJK/orbML946oRmQYbN9Hgf9RVZcNa38E915QfuUYt3Vjkie1c5N/PMq2lo630B3UGmC3rf9x0/3Ni/8FnE4XwkuT/O7WzklelmSsQ/QHlKr6Ad2R3ylM0SmiCaplA90pQuhOx+6T5FCAdr1gpDMa3wBe2vocAdy8dRQBLE6ySztNcgTdETTAwnS3xnkQ3WndbwEXAM9K8vi2rd2SPKHVMTfJ49q62z0HP0ZbQ+/mdm3iOPiPUzu3JTmsLe9fgzkHeH3vmsMTkjw0yf7ATVX1IeAjwNPa8tCd1rp2W4UYBttRVZur6n0jLHo38PdJvk03/BuLZcAXgK8CW3rt7wI+3YaGO/Itc39j7Ujpj4Hjk1wNfJ9uqP62dqF4CfCedB8t3UB3vvbWUTf4ANPeGPsj0ZV0pzQ+N6zrkUk29x7PmMa1nE33pk37uycvBv4xyb8B5zLyyPldwIIkl9IFUP+gam3b5gXA31TV1ptjnt/6Xk53avZzVTVEd3C3sm3rAuCJVfVrutNCZ7cLyBNyYb6qbqEbDVwGfJ57gwq6awOnJTmfbjTw89b+YeAK4HvtovL/oTv7cARwSZKL6U6ZbX3fejpwQVXdta1avB2FtANrF2k/VFULf5tqaW+4x7Q3y0G28y7g9qp6z7D2I4D/XlXHjLDatJBk96q6vU0vA2ZV1RvHsZ33Aauras22+jkykHZQSV5Hd/T99t/CWt4MPGaCtrWjOnrrx0TpRsR/O87tXL69IABHBpIkHBlIkjAMJEkYBpIkDANJEoaBJAn4/w+kwU/t2br/AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "labels = agg.index.to_list()\n",
    "values = agg.to_numpy()\n",
    "plt.bar(np.arange(4), values)\n",
    "plt.xticks(np.arange(4), labels)\n",
    "plt.title(\"words chosen\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The words chosen by the users stem mostly from the Object Centric (OC) model, however both JVL implementations are not far behind. Common words from several models were fused (therefore the percentages do not add up to 1.)\n",
    "\n",
    "In about 13 % of the cases, the subjects chose to use their own description instead of one proposed by the annotation modules."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Distribution of ratings:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rating_num</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rating</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>mostly well</th>\n",
       "      <td>1015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>not at all</th>\n",
       "      <td>375</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>to some extend</th>\n",
       "      <td>774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>very well</th>\n",
       "      <td>436</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                rating_num\n",
       "rating                    \n",
       "mostly well           1015\n",
       "not at all             375\n",
       "to some extend         774\n",
       "very well              436"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_frame[['rating', 'rating_num']].groupby(['rating']).count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\"mostly well\" (2 out of 3) was the most frequently chosen rating."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Print the mean rating of each source:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rating_num</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>source_file</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>awa_Shap</th>\n",
       "      <td>1.340</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>awa_kMeans</th>\n",
       "      <td>1.442</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>awa_mixin10</th>\n",
       "      <td>0.895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>awa_mixin5</th>\n",
       "      <td>1.100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>places365_Shap</th>\n",
       "      <td>2.220</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>places365_kMeans</th>\n",
       "      <td>2.196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>places365_mixin10</th>\n",
       "      <td>0.900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>places365_mixin5</th>\n",
       "      <td>1.450</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   rating_num\n",
       "source_file                  \n",
       "awa_Shap                1.340\n",
       "awa_kMeans              1.442\n",
       "awa_mixin10             0.895\n",
       "awa_mixin5              1.100\n",
       "places365_Shap          2.220\n",
       "places365_kMeans        2.196\n",
       "places365_mixin10       0.900\n",
       "places365_mixin5        1.450"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_frame.groupby(['source_file']).mean()[[\"rating_num\"]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run the coherence assessors and compute correlations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this section, we compute some of our coherence assessors and compare their results to the human ratings."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Before we can run the visual embeddings, we need to obtain the images for the concepts. Therefore, please download the respective datasets from their sources,\n",
    "* Places 365 (only validation images are needed (2.2G) from http://data.csail.mit.edu/places/places365/val_large.tar)\n",
    "* AwA2 https://cvml.ist.ac.at/AwA2/AwA2-data.zip (13G)\n",
    "and extract them. \n",
    "\n",
    "Then set the following variables to the location of the datasets. Make sure the images are in subfolders of the locations passed, to the script will find them."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "places_path = \"/ssd/tobias/places365large\"\n",
    "awa_path = \"/ssd/tobias/datasets/awa\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from add_images_script import add_images_to_concepts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting files in path: /ssd/tobias/places365large\n",
      "Collecting files in path: /ssd/tobias/datasets/awa\n",
      "Paths scanned. Found 116173 files.\n",
      "Processed  data/concepts_wo_img/places365_mixin5/concept_7.pickle\n",
      "Processed  data/concepts_wo_img/places365_mixin5/concept_11.pickle\n",
      "Processed  data/concepts_wo_img/places365_mixin5/concept_17.pickle\n",
      "Processed  data/concepts_wo_img/places365_mixin5/concept_16.pickle\n",
      "Processed  data/concepts_wo_img/places365_mixin5/concept_4.pickle\n",
      "Processed  data/concepts_wo_img/places365_mixin5/concept_10.pickle\n",
      "Processed  data/concepts_wo_img/places365_mixin5/concept_9.pickle\n",
      "Processed  data/concepts_wo_img/places365_mixin5/concept_18.pickle\n",
      "Processed  data/concepts_wo_img/places365_mixin5/concept_14.pickle\n",
      "Processed  data/concepts_wo_img/places365_mixin5/concept_1.pickle\n",
      "Processed  data/concepts_wo_img/places365_mixin5/concept_3.pickle\n",
      "Processed  data/concepts_wo_img/places365_mixin5/concept_12.pickle\n",
      "Processed  data/concepts_wo_img/places365_mixin5/concept_13.pickle\n",
      "Processed  data/concepts_wo_img/places365_mixin5/concept_19.pickle\n",
      "Processed  data/concepts_wo_img/places365_mixin5/concept_6.pickle\n",
      "Processed  data/concepts_wo_img/places365_mixin5/concept_8.pickle\n",
      "Processed  data/concepts_wo_img/places365_mixin5/concept_2.pickle\n",
      "Processed  data/concepts_wo_img/places365_mixin5/concept_5.pickle\n",
      "Processed  data/concepts_wo_img/places365_mixin5/concept_15.pickle\n",
      "Processed  data/concepts_wo_img/places365_mixin5/concept_0.pickle\n",
      "Processed  data/concepts_wo_img/places365_Shap/concept_7.pickle\n",
      "Processed  data/concepts_wo_img/places365_Shap/concept_11.pickle\n",
      "Processed  data/concepts_wo_img/places365_Shap/concept_17.pickle\n",
      "Processed  data/concepts_wo_img/places365_Shap/concept_16.pickle\n",
      "Processed  data/concepts_wo_img/places365_Shap/concept_4.pickle\n",
      "Processed  data/concepts_wo_img/places365_Shap/concept_10.pickle\n",
      "Processed  data/concepts_wo_img/places365_Shap/concept_34.pickle\n",
      "Processed  data/concepts_wo_img/places365_Shap/concept_28.pickle\n",
      "Processed  data/concepts_wo_img/places365_Shap/concept_9.pickle\n",
      "Processed  data/concepts_wo_img/places365_Shap/concept_26.pickle\n",
      "Processed  data/concepts_wo_img/places365_Shap/concept_27.pickle\n",
      "Processed  data/concepts_wo_img/places365_Shap/concept_18.pickle\n",
      "Processed  data/concepts_wo_img/places365_Shap/concept_22.pickle\n",
      "Processed  data/concepts_wo_img/places365_Shap/concept_14.pickle\n",
      "Processed  data/concepts_wo_img/places365_Shap/concept_39.pickle\n",
      "Processed  data/concepts_wo_img/places365_Shap/concept_33.pickle\n",
      "Processed  data/concepts_wo_img/places365_Shap/concept_25.pickle\n",
      "Processed  data/concepts_wo_img/places365_Shap/concept_1.pickle\n",
      "Processed  data/concepts_wo_img/places365_Shap/concept_37.pickle\n",
      "Processed  data/concepts_wo_img/places365_Shap/concept_29.pickle\n",
      "Processed  data/concepts_wo_img/places365_Shap/concept_3.pickle\n",
      "Processed  data/concepts_wo_img/places365_Shap/concept_23.pickle\n",
      "Processed  data/concepts_wo_img/places365_Shap/concept_30.pickle\n",
      "Processed  data/concepts_wo_img/places365_Shap/concept_12.pickle\n",
      "Processed  data/concepts_wo_img/places365_Shap/concept_20.pickle\n",
      "Processed  data/concepts_wo_img/places365_Shap/concept_13.pickle\n",
      "Processed  data/concepts_wo_img/places365_Shap/concept_36.pickle\n",
      "Processed  data/concepts_wo_img/places365_Shap/concept_19.pickle\n",
      "Processed  data/concepts_wo_img/places365_Shap/concept_6.pickle\n",
      "Processed  data/concepts_wo_img/places365_Shap/concept_8.pickle\n",
      "Processed  data/concepts_wo_img/places365_Shap/concept_31.pickle\n",
      "Processed  data/concepts_wo_img/places365_Shap/concept_24.pickle\n",
      "Processed  data/concepts_wo_img/places365_Shap/concept_2.pickle\n",
      "Processed  data/concepts_wo_img/places365_Shap/concept_32.pickle\n",
      "Processed  data/concepts_wo_img/places365_Shap/concept_5.pickle\n",
      "Processed  data/concepts_wo_img/places365_Shap/concept_15.pickle\n",
      "Processed  data/concepts_wo_img/places365_Shap/concept_38.pickle\n",
      "Processed  data/concepts_wo_img/places365_Shap/concept_35.pickle\n",
      "Processed  data/concepts_wo_img/places365_Shap/concept_21.pickle\n",
      "Processed  data/concepts_wo_img/places365_Shap/concept_0.pickle\n",
      "Processed  data/concepts_wo_img/places365_mixin10/concept_7.pickle\n",
      "Processed  data/concepts_wo_img/places365_mixin10/concept_11.pickle\n",
      "Processed  data/concepts_wo_img/places365_mixin10/concept_17.pickle\n",
      "Processed  data/concepts_wo_img/places365_mixin10/concept_16.pickle\n",
      "Processed  data/concepts_wo_img/places365_mixin10/concept_4.pickle\n",
      "Processed  data/concepts_wo_img/places365_mixin10/concept_10.pickle\n",
      "Processed  data/concepts_wo_img/places365_mixin10/concept_9.pickle\n",
      "Processed  data/concepts_wo_img/places365_mixin10/concept_18.pickle\n",
      "Processed  data/concepts_wo_img/places365_mixin10/concept_14.pickle\n",
      "Processed  data/concepts_wo_img/places365_mixin10/concept_1.pickle\n",
      "Processed  data/concepts_wo_img/places365_mixin10/concept_3.pickle\n",
      "Processed  data/concepts_wo_img/places365_mixin10/concept_12.pickle\n",
      "Processed  data/concepts_wo_img/places365_mixin10/concept_13.pickle\n",
      "Processed  data/concepts_wo_img/places365_mixin10/concept_19.pickle\n",
      "Processed  data/concepts_wo_img/places365_mixin10/concept_6.pickle\n",
      "Processed  data/concepts_wo_img/places365_mixin10/concept_8.pickle\n",
      "Processed  data/concepts_wo_img/places365_mixin10/concept_2.pickle\n",
      "Processed  data/concepts_wo_img/places365_mixin10/concept_5.pickle\n",
      "Processed  data/concepts_wo_img/places365_mixin10/concept_15.pickle\n",
      "Processed  data/concepts_wo_img/places365_mixin10/concept_0.pickle\n",
      "Processed  data/concepts_wo_img/awa_mixin5/concept_7.pickle\n",
      "Processed  data/concepts_wo_img/awa_mixin5/concept_11.pickle\n",
      "Processed  data/concepts_wo_img/awa_mixin5/concept_17.pickle\n",
      "Processed  data/concepts_wo_img/awa_mixin5/concept_16.pickle\n",
      "Processed  data/concepts_wo_img/awa_mixin5/concept_4.pickle\n",
      "Processed  data/concepts_wo_img/awa_mixin5/concept_10.pickle\n",
      "Processed  data/concepts_wo_img/awa_mixin5/concept_9.pickle\n",
      "Processed  data/concepts_wo_img/awa_mixin5/concept_18.pickle\n",
      "Processed  data/concepts_wo_img/awa_mixin5/concept_14.pickle\n",
      "Processed  data/concepts_wo_img/awa_mixin5/concept_1.pickle\n",
      "Processed  data/concepts_wo_img/awa_mixin5/concept_3.pickle\n",
      "Processed  data/concepts_wo_img/awa_mixin5/concept_12.pickle\n",
      "Processed  data/concepts_wo_img/awa_mixin5/concept_13.pickle\n",
      "Processed  data/concepts_wo_img/awa_mixin5/concept_19.pickle\n",
      "Processed  data/concepts_wo_img/awa_mixin5/concept_6.pickle\n",
      "Processed  data/concepts_wo_img/awa_mixin5/concept_8.pickle\n",
      "Processed  data/concepts_wo_img/awa_mixin5/concept_2.pickle\n",
      "Processed  data/concepts_wo_img/awa_mixin5/concept_5.pickle\n",
      "Processed  data/concepts_wo_img/awa_mixin5/concept_15.pickle\n",
      "Processed  data/concepts_wo_img/awa_mixin5/concept_0.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_7.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_11.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_17.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_42.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_16.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_4.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_46.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_10.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_34.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_28.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_9.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_26.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_27.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_18.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_22.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_47.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_14.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_39.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_33.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_43.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_25.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_1.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_37.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_29.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_3.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_48.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_23.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_30.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_44.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_12.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_20.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_45.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_41.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_13.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_36.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_19.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_40.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_6.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_8.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_31.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_24.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_2.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_32.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_5.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_15.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_49.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_38.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_35.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_21.pickle\n",
      "Processed  data/concepts_wo_img/awa_kMeans/concept_0.pickle\n",
      "Processed  data/concepts_wo_img/awa_Shap/concept_7.pickle\n",
      "Processed  data/concepts_wo_img/awa_Shap/concept_11.pickle\n",
      "Processed  data/concepts_wo_img/awa_Shap/concept_17.pickle\n",
      "Processed  data/concepts_wo_img/awa_Shap/concept_16.pickle\n",
      "Processed  data/concepts_wo_img/awa_Shap/concept_4.pickle\n",
      "Processed  data/concepts_wo_img/awa_Shap/concept_10.pickle\n",
      "Processed  data/concepts_wo_img/awa_Shap/concept_34.pickle\n",
      "Processed  data/concepts_wo_img/awa_Shap/concept_28.pickle\n",
      "Processed  data/concepts_wo_img/awa_Shap/concept_9.pickle\n",
      "Processed  data/concepts_wo_img/awa_Shap/concept_26.pickle\n",
      "Processed  data/concepts_wo_img/awa_Shap/concept_27.pickle\n",
      "Processed  data/concepts_wo_img/awa_Shap/concept_18.pickle\n",
      "Processed  data/concepts_wo_img/awa_Shap/concept_22.pickle\n",
      "Processed  data/concepts_wo_img/awa_Shap/concept_14.pickle\n",
      "Processed  data/concepts_wo_img/awa_Shap/concept_39.pickle\n",
      "Processed  data/concepts_wo_img/awa_Shap/concept_33.pickle\n",
      "Processed  data/concepts_wo_img/awa_Shap/concept_25.pickle\n",
      "Processed  data/concepts_wo_img/awa_Shap/concept_1.pickle\n",
      "Processed  data/concepts_wo_img/awa_Shap/concept_37.pickle\n",
      "Processed  data/concepts_wo_img/awa_Shap/concept_29.pickle\n",
      "Processed  data/concepts_wo_img/awa_Shap/concept_3.pickle\n",
      "Processed  data/concepts_wo_img/awa_Shap/concept_23.pickle\n",
      "Processed  data/concepts_wo_img/awa_Shap/concept_30.pickle\n",
      "Processed  data/concepts_wo_img/awa_Shap/concept_12.pickle\n",
      "Processed  data/concepts_wo_img/awa_Shap/concept_20.pickle\n",
      "Processed  data/concepts_wo_img/awa_Shap/concept_13.pickle\n",
      "Processed  data/concepts_wo_img/awa_Shap/concept_36.pickle\n",
      "Processed  data/concepts_wo_img/awa_Shap/concept_19.pickle\n",
      "Processed  data/concepts_wo_img/awa_Shap/concept_6.pickle\n",
      "Processed  data/concepts_wo_img/awa_Shap/concept_8.pickle\n",
      "Processed  data/concepts_wo_img/awa_Shap/concept_31.pickle\n",
      "Processed  data/concepts_wo_img/awa_Shap/concept_24.pickle\n",
      "Processed  data/concepts_wo_img/awa_Shap/concept_2.pickle\n",
      "Processed  data/concepts_wo_img/awa_Shap/concept_32.pickle\n",
      "Processed  data/concepts_wo_img/awa_Shap/concept_5.pickle\n",
      "Processed  data/concepts_wo_img/awa_Shap/concept_15.pickle\n",
      "Processed  data/concepts_wo_img/awa_Shap/concept_38.pickle\n",
      "Processed  data/concepts_wo_img/awa_Shap/concept_35.pickle\n",
      "Processed  data/concepts_wo_img/awa_Shap/concept_21.pickle\n",
      "Processed  data/concepts_wo_img/awa_Shap/concept_0.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_7.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_11.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_17.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_42.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_16.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_4.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_46.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_10.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_34.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_28.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_9.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_26.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_27.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_18.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_22.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_47.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_14.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_39.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_33.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_43.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_25.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_1.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_37.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_29.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_3.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_48.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_23.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_30.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_44.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_12.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_20.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_45.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_41.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_13.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_36.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_19.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_40.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_6.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_8.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_31.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_24.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_2.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_32.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_5.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_15.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_49.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_38.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_35.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_21.pickle\n",
      "Processed  data/concepts_wo_img/places365_kMeans/concept_0.pickle\n",
      "Processed  data/concepts_wo_img/awa_mixin10/concept_7.pickle\n",
      "Processed  data/concepts_wo_img/awa_mixin10/concept_11.pickle\n",
      "Processed  data/concepts_wo_img/awa_mixin10/concept_17.pickle\n",
      "Processed  data/concepts_wo_img/awa_mixin10/concept_16.pickle\n",
      "Processed  data/concepts_wo_img/awa_mixin10/concept_4.pickle\n",
      "Processed  data/concepts_wo_img/awa_mixin10/concept_10.pickle\n",
      "Processed  data/concepts_wo_img/awa_mixin10/concept_9.pickle\n",
      "Processed  data/concepts_wo_img/awa_mixin10/concept_18.pickle\n",
      "Processed  data/concepts_wo_img/awa_mixin10/concept_14.pickle\n",
      "Processed  data/concepts_wo_img/awa_mixin10/concept_1.pickle\n",
      "Processed  data/concepts_wo_img/awa_mixin10/concept_3.pickle\n",
      "Processed  data/concepts_wo_img/awa_mixin10/concept_12.pickle\n",
      "Processed  data/concepts_wo_img/awa_mixin10/concept_13.pickle\n",
      "Processed  data/concepts_wo_img/awa_mixin10/concept_19.pickle\n",
      "Processed  data/concepts_wo_img/awa_mixin10/concept_6.pickle\n",
      "Processed  data/concepts_wo_img/awa_mixin10/concept_8.pickle\n",
      "Processed  data/concepts_wo_img/awa_mixin10/concept_2.pickle\n",
      "Processed  data/concepts_wo_img/awa_mixin10/concept_5.pickle\n",
      "Processed  data/concepts_wo_img/awa_mixin10/concept_15.pickle\n",
      "Processed  data/concepts_wo_img/awa_mixin10/concept_0.pickle\n",
      "Successfully added images to 260 concepts\n"
     ]
    }
   ],
   "source": [
    "ret = add_images_to_concepts(\"data/concepts_wo_img\", [places_path, awa_path])\n",
    "print(\"Successfully added images to\", ret, \"concepts\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Assessors: Get Visual-Embeddings (VE) and Best-Aligned Meaning (BAM) Scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded 260 concept examples and concept activation maps.\n"
     ]
    }
   ],
   "source": [
    "eval_concepts = []\n",
    "concept_model = ['awa_Shap', 'awa_kMeans', 'awa_mixin5', 'awa_mixin10',\n",
    "                 'places365_Shap', 'places365_kMeans', 'places365_mixin5', 'places365_mixin10'] #\n",
    "for m in concept_model:\n",
    "    foldername = \"data/concepts_wo_img/\" + m\n",
    "    eval_concepts.extend(read_concepts(foldername))\n",
    "print(f\"Loaded {len(eval_concepts)} concept examples and concept activation maps.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Name the annotation outputs\n",
    "words_dictOC = words_lists[0]\n",
    "words_dictJVL = words_lists[1]\n",
    "words_dictJVLCrop = words_lists[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a whole matrix for results of the coherence assessment modules.\n",
    "# First, we test Visual Embeddings (VE) with different coherence metrics and the Best-Aligned Meaning (BAM) module with different annotators.\n",
    "from coherence_assessor import BestAlignedMeaningAssessor, VisualEmbeddingsAssessor, HybridAssessor\n",
    "import torchvision.models as models\n",
    "\n",
    "use_device = \"cuda:1\"\n",
    "\n",
    "model = models.resnet50(pretrained=True).to(use_device)\n",
    "# Set up a dict with the names and the corresponding annotation modules.\n",
    "columns = [(\"ve_dmin\", words_dictOC, VisualEmbeddingsAssessor(device=use_device, coherence_measure = \"min\", cam_weighted=False, model=model)),\n",
    "           (\"ve_dmean\", words_dictOC, VisualEmbeddingsAssessor(device=use_device, coherence_measure = \"mean\", cam_weighted=False, model=model)),\n",
    "           (\"ve_dmin_weigh\", words_dictOC, VisualEmbeddingsAssessor(device=use_device, coherence_measure = \"min\", cam_weighted=True, model=model)),\n",
    "           (\"ve_dmean_weigh\", words_dictOC, VisualEmbeddingsAssessor(device=use_device, coherence_measure = \"mean\", cam_weighted=True, model=model)),\n",
    "           (\"bam_jvl\", words_dictJVL, BestAlignedMeaningAssessor()),\n",
    "           (\"bam_jvl_crop\", words_dictJVLCrop, BestAlignedMeaningAssessor()),\n",
    "           (\"bam_oc\", words_dictOC, BestAlignedMeaningAssessor()),\n",
    "          ] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Some space to save results.\n",
    "results_dict = {c[0]: np.zeros(len(eval_concepts)) for c in columns}            \n",
    "results_dict[\"ground_truth\"] = np.zeros(len(eval_concepts))\n",
    "results_dict[\"folds\"] = np.zeros(len(eval_concepts))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['ve_dmin', 've_dmean', 've_dmin_weigh', 've_dmean_weigh', 'bam_jvl', 'bam_jvl_crop', 'bam_oc', 'ground_truth', 'folds'])"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_dict.keys()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load the pretrained ResNet50 for the visual embeddings and register a hook to access the latent representation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_average_score(file_name, source_id):\n",
    "    selection = data_frame[(data_frame['source_file'] == file_name) & (data_frame['source_id']==source_id)] #\n",
    "    return selection[\"rating_num\"].mean()\n",
    "\n",
    "def get_meaning_from_list(wordlist, file_name, source_id):\n",
    "    for item in wordlist:\n",
    "        if item[\"source_file\"] == file_name and item[\"source_id\"] == source_id:\n",
    "            return item[\"word_list\"]\n",
    "    return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'word': 'head', 'score': 1.9661644970155396},\n",
       " {'word': 'ear', 'score': 1.784575711716616},\n",
       " {'word': 'grass', 'score': 1.7227018854713114},\n",
       " {'word': 'bear', 'score': 1.4485389878489097},\n",
       " {'word': 'fur', 'score': 1.0509405324383936},\n",
       " {'word': 'tree', 'score': 1.0062782881996035}]"
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_meaning_from_list(words_dictOC, \"awa_Shap\", 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "260it [01:54,  2.28it/s]\n"
     ]
    }
   ],
   "source": [
    "source_to_idx = {} # A dictionary mapping the (source_file, source_id) -> index in the results array.\n",
    "from tqdm import tqdm\n",
    "for i, my_concept in tqdm(enumerate(eval_concepts)):\n",
    "    # Load concept and compute CNN features\n",
    "    \n",
    "    results_dict[\"folds\"][i] = split_dict[(my_concept.source_file, my_concept.source_id)]\n",
    "    results_dict[\"ground_truth\"][i] = get_average_score(my_concept.source_file, my_concept.source_id)\n",
    "    \n",
    "    for name, annotations, assessor_obj in columns:\n",
    "        #print(name, len(annotations), assessor_obj)\n",
    "        #retrieve annotations\n",
    "        meaning = get_meaning_from_list(annotations, my_concept.source_file, my_concept.source_id)\n",
    "        results_dict[name][i] = assessor_obj.assess(meaning, my_concept)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Visualize the results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For nice latex figures. Skip if you are happy with normal output.\n",
    "plt.rcParams.update({\n",
    "    \"text.usetex\": True,\n",
    "    \"font.family\": \"sans-serif\",\n",
    "    #\"font.sans-serif\": [\"Helvetica\"]}\n",
    "    \"text.usetex\": True,\n",
    "    \"pgf.texsystem\": \"pdflatex\",\n",
    "    \"pgf.preamble\": r\"\\usepackage{bm,times}\",\n",
    "    \"font.family\": \"times\",\n",
    "    # Use LaTeX default serif font.\n",
    "    \"font.serif\": [\"Times\"],\n",
    "    # Use specific cursive fonts.\n",
    "    \"font.cursive\": [\"Comic Neue\", \"Comic Sans MS\"],\n",
    "\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAADQCAYAAABStPXYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABRU0lEQVR4nO29e3Rc93Xf+9kH7yeHMwBISRQIDiQ+pNiRAVAP223aCsijdeK2IULbubXTNgSS9qaraRtCsp20aR1ToHNzu9p7bwLQrZN0RRJJJI0TO+01oCa+9bJEAhgpVSRKlAiKIEVZBGYwJN6v87t/nHMGBzPnAIPHYAbA77MW1sx5zDm/Gcw5e/b+7f3dopRCo9FoNJpcw8j2ADQajUaj8UIbKI1Go9HkJNpAaTQajSYn0QZKo9FoNDmJNlAajUajyUm0gdJoNBpNTpKf7QFoNo6IhJVSQ8nPN/H4DcBLQONKx87EuTWalRCRZqAX6LMfjwP9Sqmzrn06gSgQBxqBLqVUJOk4nUAAuAaEgAagUynVl/l3ofFDG6htjoiEgS6gxf18M8+hlIqISCzdcWzmuTWalVBK9YlIBLiolOoGEJExEYnY23rd2+zt10SkQynVYy/3AhGlVLtrn9Nb/FY0HugQX44gIqdFpE1Eulzr2kTkhIh02b8UEZFOEWkWkYv2bs1Ak4i0JT13Xn9aRHpFJGwvD9rrrvmMYdD1eDF5n6RxdTrnSj63RpNlwvY10+w2Tjad9h8icgJoUkp1uHewPbCUH2XJ16S9fM21TbmeJ641+zXK/iFH0jW97Dq116XcD3YlSin9l+U/rJv7aft5g2tdp/s5Vtihy17XBZwAwsA1e537+QnghP28094/DCisUEbYYxxhYMy1fM11jGv29masX6TufRrc59Z/+m8r/4BB+zveDFwEeu31nV7fSXs/5zroAgbTPI/XNbnse2/dUhPX0rJrzR5ns/3cud69rtOU+8Fu/dMhvtxgALgoIiG19EuuFetiQ1lxcCcW3u7ylnpXOGYLcM3e9xowhPWLMK6UimPF45OJsfxXYw9WTL/Hta4V6Hct9wEnsS4sjSZbBIAOrO93psLMKdek4/F44HWtdQHtWNeMs87rOvW6H+xKdIgvB7C/xIeABhEZtFcHsS66BCISsOPlboPlRxArrt5t/613sjfqsS7keh732Uej2UoGsQxIsyvM3IvlySTTwJLxGLSX0yHlmlwLygo1OuO74DrmsuvU536wK9EGKgcQkRNKqeRffr3As6592oA2YEhZmXJB175ez/ux4+yu16eD+1gNLPeewPoFecK1HHbtE0SjyRL2jf1poEtEGlxeTnLCQztwyn5NNzBkZ/ElsOdsk42b1zUZw/7eu+aPVjJ4fUCLPVbwuE597ge7Em2gcgTXpGkXLLtwxuxkhQtYX+5m+2IawrrQYixdYO7n3UDMfn2v/fqfAQLO5KwPAWeCFmu+a8i+4MJYsfI+rBtApz3BfF4p5RjNIXt9YLM/H43GC/u7HAZa7TKHCFao7yURabZv8qGkpIN2ZWfwASil6rG+9xddiT8pJRNe16RtaC7Y11gz1nXZhP+1dgZXOFzZyRhJ12nK/WC3IvZEnEaDbVgG7QtWo9Fosor2oDRumrDScwPZHohGo9FoD0qTwFVzEd/t2UMajSb7aAOl0Wg0mpxEh/g0Go1Gk5NoA6XRaDSanCTnlSSqqqpUXV1dtoeh0SxjcHBwVClVne1xpIO+hjS5ymrXUc4bqLq6OgYGBrI9DI1mGSJyI9tjSBd9DWlyldWuIx3i02g0Gk1Oog2URqPRaHISbaA0uw/ThIk7kMMlFiLSYEvf+G13+oLpxnqaHUvOz0F5MT8/z61bt5iZmcn2UDQZori4mAMHDlBQULC5BzZN+L1Pwc1L8OAT8IVvgZF7v9OU1cW4yWubrYF4TVkdYxtscdFkUV+NxpfnLw3zzdfez+g5vnD3dwiWF/LkPzm37mNsSwN169YtKioqqKurQ0SyPRzNJqOUIhqNcuvWLQ4dOrS5B58atYyTuWA9To1Cec3mnmPzSOnoatPCkohoBKvNxDID5VK/p7a2NlPj02xTvvna+7z5wT0eua8yY+c4uHCNsqmNmZhtaaBmZma0cdrBiAihUIiRkZHNP3hZteU5OR5U2bbIFE/G3QYihkfPI1t5uxugqakpd2OZmrTYbI/HMU7n25/a2IEGvgGv+zjvMgyhj2zo8NvSQAHaOO1wMvb/FbHCelOjlnHant8jdxuIYNKyZhuymgG6dN1ypp84tDkt1x65r5JPP/bAxg/0eg/84HXY72GI9n8EPnIidf0a2LYGSrN1xONxBgYGaGpqIhAIZHs4G8cwcjmslw69WMrzEaymkhezO5ydT6bnbFYzQE8cCvLpxx7gc09kKVzr5yk5xukffjsjp9UGah309PTQ2tpKb28vzc1WP7L29nbq6+sJh8O0trbS2dlJIBDg2rVr1NfX09bm3dC2vb2drq7c7kkWCATo6OhgcHBXd5/eUpwmkXZn2Ii9rlcp1aKU6nGy+ICQ3URSs0FWMkKb7cEkk3UD5OBniG58z3o8+Mnl6zfBS1oJbaDWwYkTJ2hubmZoaCmyEggEOH16KePX/dy9n5uhoSEuXLhAa2trwtDlKsGg7ua+ldhGSZLWtbieO+1QtHHaJFZKHMgZA7LZJBskP0N08JOWIWr6h1s3NnaRgTJNRXRyjqrywk2Z3+jo6KCjoyPhGYVCIc/9uru7fb2nnp4eOjs76erqShiooaEhIpEIvb29dHZ2EovFli0DXLhwgcHBQRobG2lubl51/+7ubsLhML29vXR1ddHd3c3g4CBDQ0N0dXURDqfMsQMQiUTo6OggFovlvAHVaFYinRDdpiUO5BIrJTFAqkHKkiHyY1cYKNNUfPbcKwzeGKPx4F5eOPUkhrExI9Xc3Ew8HicSiTAwMMCJE8vd3O7ubuLxOP39/b4GCqCtrY2Ojg7i8TiBQIC+vj7i8XgiRHjhwoVlyx0dHdTX19PY2EhHRwednZ2r7t/S0kJzczP9/f2cPXuWhoYGent76e31rQMlHo/T2trK4OAgZ86c4eTJkxv6vDSatbKZ8z7phOg2LXEgF3AMk59H5JBjBimZXWGgopNzDN4YY8FUDN4YIzo5R3VF0YaP29HRwZkzZwiHwylGyFl2wnuO1xIIBOjs7KSvr49oNJrwbrq7uzl9+jRtbW20tLTQ39/PxYsXU5YjkUjCM3LOsdL+7vBiS0tLwltbLWR35swZOjo6CAQCDA0N0dDQQF9fH5FIhGg0ysmTJ+nq6qKrq4uenh76+/vp7OyktbWVSCSSmFdz9m9vb/f11DQaLzazVmfHhuiS8TJMOWyAViMrBsqWZzkO9Culzmb6fFXlhTQe3JvwoKrKCzfluI73c+6cf6V0OBymp6cnxYC5DU1TUxNPP/00p0+fZmhoiN7eXjo6Oujp6Ul4O85yIBCgp6eHEydOJIzPSvuHw2EikUhizqylpSVljF709fXx7LPPMjQ0RDweByxjGw6HuXbtWiKbr7u7m2AwSH19PWfPnqWlpYWWlhaampro6+tLGCVtnDQr4eUt7ciQ22biFb7bIYbJYcsNlJ2VdNZ+Pghk3ECJCC+cenJT56Ac2traloX3enqsL4xz4+7v7ycejy/bp7u7m2vXriWWnZu3E74DqK+vp7m5mQsXLixbbmhooKWlhd7eXlpbWxNGym//EydO0N7enggdnj59mrNnzzIwMLAsrNjZ2bks5Nfc3Ex3dzcNDQ2AZZwGBwfp6uqit7eXcDhMIBCgt7eXc+fOEQgEaG1t5eLFiwnPy5kHO3PmzKZ93pqdiZe3tKNCbpnAqwZphxgmhy03UK6U2QaW5FoyjmHIpoT1knG8IIcTJ06gXCKkyXNTQIo3FQgEGBsb8zy+175u47ba/kAi3OYkOpw+fXpZlqEzn+bG/b6c1zleU319fcIwHj9+nL6+PhoaGggGg0QiEeLxOENDQwQCgcRrHGOo2Z2sNp+kvaU1MvANy1s6+MmM1SDlAqKyoOgsIgHgHNCglKr32O7WEWu8cWN5T6srV65w7NixLRipJpts+f/ZNNNWmBCRQaWUp5hrrtHU1KSy2bDw+UvDfPG/vg6snKSwK+aIVmK1jDs3TijvU/9+W3tLq11HWZmDUkrFgVYRueguRHRt1zpimq1lm6icbyccr8nJoPvq3/vI7jVA6Rif1TLu3OywUJ4f2c7iG0LriGlyge2lcr4tcOaVdk0GXTJuo5SO8dklRmctZCNJ4gRWu4BeoNf2pjSa7LIzVM5zhucvDXPpeownDgV317ySn1HSxmddZCNJooek3jUaTdbZGSrnWSc5rLejs/BWS/PWRmnDZDvEp9HkDttf5Tzr7Jqw3sA34Fv/3HruDttpo7SpaAOl0Wg2lV2RLu54Tts8iy7X0WlK66Cvrw8R4ezZs3R0dNDa2ppSR9Te3u75mr6+vpT99u7d66t47nWsXMMpTnbEZXt6ejh79uyy9xqJRGhvb6e7u5vu7m4aGxsTWobJn51Gsy04+EltnDKM9qDWQXNzM4FAgLa2NgKBQOLG6xS/erXRcFQgOjs7lymDDw0N0dTU5CsFlOstOc6ePcvp06cT4rLuIuLGxsZlKhPJBcaxWIyGhoYVFd8zwhrqnTTp406M2BGslBru10VWs6nsHg/KNGHiDmSgMNkRU3Vwt9Fwc/LkSQYGBpYJyK7mHXkda2hoiJ6eHtrb2xOqDe7leDyeOHZ3d3da+589ezaxzj22lpaWFftZRaPRxP7JBvTkyZOcOXOGrq6uFEWNtlOnCB98EJSiqamJs2czrnhl4dQ7/dYx+N2/Yy1rNgVHKWLHJEY4UkJeZLhRn8ZidxioDN2Uzpw5Q2trK7FYjKam5cXQbW1tCf07h0AgwLPPPpuQEUpX/if5WH19fQwNDSVaaiQvO9p3TkuOdPZvaGjgxIkTBAIBzp49SzgcJhaLJXT3vOjr6+P48eMA9Pf3p7yXcDic6G+1rF+WUhB9Fz58A6LvEj50aMXWH5uKV72TZtN44lBw+yZGDHwDvvF3lv7c7cy9/nR4L+PsDgOVoZvSs88+y8WLFzl58iSnTp0C8Gyj4aatrY0LFy7Q09OT4nE4XktHR8eKx2pra6O3tzdxzuTlSCRCW1sbbW1tjI2Nrbp/ckuO/v5+gsHgqi053MY3HA6nzCU56ueOAnoCcwHmJgEFc5MEKsuJxWIrnmvTcOqdjHxd77RBnr80zMmulxN/b35wL9tDWh+OYfrWP19KEwftJeUAu2MOKsNFmOFwmPPnzwP+bTRgyWNqbm5OqIK7Ewm2W0uOhoaGxPGdcKAb5xzBYJCnn3464bFh5BOfXoC5KQLV9xG/N7F17Th0vdOm4KWvt63Ux/0KanWKeE6xOwzUJt+UnHBbd3c3gUCAixcvcu7cuRXbaBw/fpzz58/T1taW6LMEcPHixcS8lPsmvV1acjjeXjgcTrTaqK+vJx6P097envASL168yKlTpzh+/DjhQ4cIGJM0P/XDgEr0nloTG0l02Cb1TiLSiaW40uDVN80WVR7y254p3MZpW+rrJdcwacOUs2RFzXwteCkxazXz7OB4ZW4ikQiBQGBtHtDivDX/hAKE7j95mbZf+MWU3Xz/zzkg7JppNXNbEiyolOq2G3wO2Sos7u1hpdRZ25Cd8ZMN22w185NdL3Ppeiz3jFO6auA7RAl8J7DadbQ75qA0m4JXbyt39mLaGPlQWAYIQ7ejtLX/wtpev9E5xQxmdG4iLYBjVSL2sps+oF1EmoHoVmta5mQyxEpZd24OflIbp21CWiE+EfkVrAtkL1ZI4YxS6rUMjkuzjVjz/JEIhB4Cc4HwfflrD9FtZE7RNFG/+ym4Zb1WcrethvtDjSUto5SK257TReBU8ouTeqplcJg5wi5p4LfbWNFAicge4Gewfq11K6Xu2uuaRaRJKfX1rRikZgciAnkF6e2bPN8kAp//Exi9CtVH12TgzIkRzOFXyGeRhRuvYEyMYFTuW+ebyCju4rNg0jIiEgYCwCFgUEQiSqnEPruup5oT2tNZdzuK1X46HlJKnVNKvaqUuguglLqrlPpDpdTXRaQu80P0JtfnzjQbI/H/9aphM034/Z+Crr9mbVtDXVuUPQyaDzOvDAbNh4myJ0PvYMP0Ak5svgHLU3LTDMTt0F6nvc/uRksP7ThWDfGJyGMeq5uUUl9XSr236SNKg+LiYqLRKKFQCNFpwjsOpRTRaJTi4mL/+aZ1Nhesqijil+7/La4P36Cuto4XK4oy+E7Wj1KqR0Q67TmmkFKqD0BEepVSLcAF4Fl7e8D2mDSaHcVqBurrWBO1zcAgMIYVbthjb8sKBw4c4NatW4yMjGRrCJoMU1xczIEDByA/33u+aZ1zUCLC820fJzrZRFV54ao/cMzFRWIjtwnVPIBs8VyVUqrDftrnWtdiP8aBlO2ZZsv09tLNyHPQ2ng7khUNlJP+JyI/rZT6Q2e9iJzJ9MBWoqCggEOHDmVzCJqtxGu+aQN1bYYhVHt5TklzXebiIm91/giHZ9/gStGjHO34LkZe3ia8oe3LmvT21mpk3KTTIt2NVn3YkaRbqBsWkb+FNVHbguVRrbGy0kJEAsA5+xjdrl+JGk0qpgm/95N21t2TSzVPm11s61FbFRu5zeHZN8gXk8OzbxAbuU3V/gc375zbFN8U82SDtFYj40YXz2pI00Appb5mp5o/A1zDyuxbL81KqVYAERkTkS539pFmF+OlDjFxB4ZfBhTc+L61XLl/88/tMdcVqnmAK0WPcnj2Da4WPcqxmm0i45MtnDokJ9SmjYxmg6RbB3UKeEYpFRKRQ1iprdfXc0J3NTwwoI2TBvBXhxDBUpzAesxUUoxHbZWIcLTju8RGbnMsC3NQuUZa80+O+rdGswmkG+JrxPKeUEpdt6VX/sdGTmyH+iI+23ZXkaHGO1uvvMb6q/34kuHIlIaej16jkZenw3o2y+afvOaXdKKCZpNJ10ANYv+MFZGfJlV2ZT20+c0/7boiQw2UhKCgFGbvWY8ldv8oEfi5b2+N+vg2EZHNJon5p2/0pBoknaig2WTSNVDngS+KyHeASawEh3Vj1250288DW60jpslBpqN2jyisx+nokrHQhiM30eE8TYZJN6j+EvCiUupHlVJ/D1i5T/kK2CrMXcBFERlkg8ZOs0Moq4baJy0h2dons9JI0DQVI+OzWqVEo8kR0vWgXgK+LiL9WMWB6y4MtJMk1lkcodmWpNO7aY09u0xTEZ2cW7XY1ms/v3WfPfcKgzfGaDy4lxdOPYlhaJUST9zCrBpNBknXg7psF+2+hJXY0LbK/hqNhZeWnh9OKC8N4/SZ7ld48sxLnOx+BdP09ngco/PUmZf4jL2f1zqA6OQcgzfGWDAVgzfGiE7Orfst72SenvqzpWZ/er5Jk2HSNVAhEalUSvUopR5inSnmml3IRns3kRp6Gxmf5fJ7MRZNxeXrMUbGZz33i07OEbkRJWDGGbwRIzo557kOoKq8kMaDe8k3hMaDe6kqL9ykD2D78/ylYU52vcybH9zjE9N/bq3U/ZQ0W0C6hbrnklZ9NQNj0exENtK7Ce/QW7KDJeK9X1VZPt8sfy5RaFtV9uMAnutEhBdOPZlW2HC38c3X3ueRD/6IXyt8mYfN97RquGbLWK0f1G8rpX5RRH4HqxYKQLAKdUOZHpwmx9nA3FK6c0heobfqiiIer9ubMEbVFUWMTnjsJ3c5tnAFEdN6nIoCpK6zMwR9Nfo0nCh8mUdlGB74YR3a02wZq3lQz9iPF4EOV8NC3x7yml2Cn/KD164IUbWHKqxfN6ap+Fz39xMtL15oe8pKSPAweFXlhTTUBhi8MUZDbSBh0J7/+Sd5d2SCw/vKEZFEiM4xWlaIrho58DjcuoTY3ptpKt7KP2Z5UPnHOFpSlXace7fx/KVhvvna+7z5wT0oRKeVa7ac1dTMnSaFL7nX2dl8mt2Mn/JDEl6ht+jEDL98+1/QUHCVyO3DRCf+guryIk+DpxSIMglxF1QApax+UT/7ny6lZNylhOichAwF9guJTs3z6Yln2GPeIz6/h5en5rXX5INjnB65r5KqWf0Zabae1UJ8z2HVKSmsH78OOsS320lzbik6OcfAjTEWTcWAHXqrknvsNd4hH5NGeYc8uQdTgrp5CTEXrEfb4I3cm+aXb/9LGguvMnj7MCP3/hwjLy/lmNUVRakhuqlRuHUZ1KL1ODVKVXk1DQdDDN4wdDLECrh19863PwXfKM72kDS7kNVCfL1KqWeSV4rIP8jQeDTbBRHvPk1JBEsLKC3MY3xmgdLCPIKlBYhRQ17tk6hbl8h78AmkvMY39GZMj9JovE2BKBqNt4lPjxKsOUB5gWLf7E0+KDhIsLTAe4w+ArA6GWJ11tT3SaPJEKuF+F4CEJG/D5x0bQoD/yWD49LkOqYJv/9TKSG55OSH2NQ8k7MLAEzOLhCzQ2ryc8sTJ6JTc/zUeAcBNc7YXCWv2PtV1dzPlFFCvjnFnFFCVc393Bmf4n/yj6kommacEkbH32VfoDx1DstPAFYnQ6SFb98njWaLSHd+uB54Dqv9+zPYOnqaXYzHHJRpKk52fZ8nfqOPn/md72OaKuFBAQkPyotgaQElhYWMsofSwoLEfjIdo1TmEMF6nI5hjLxNBdOIQAXTGCNv+xcEp1n8C1rqSKPJNdI1UHFgzH7+MUDnme4ETNNqAJjGDTnl5u2Ez4z8RPjsw/EZ+m/EMYH+G3E+HJ9hdHKOidlFACZmFxmdnPM0JqMTc0zMzlHFXcZn5xmdsApoF4pCDJgPM68MBsyHWSgKQWnS9GdpaMMFwX4KE7sNd1GuRpNt0tXiuwAEXZ11tQe13VlLmrgtLTQ4bGXNvejo1CWFz+JJ8kDxyTlC5ctDaQKexkSpMl4o+AqNxlUGzcMo9TcBeHd0kp+Z+RIhxhmlkv8+OsmR/QeYkmJK1QxTUkzV/gNg5G2oINiv3iqbiEgn0As0KKXO+uzTBsSAvs3oCuDO3EvMP2ntPU2WSMuDUkrdVUpdt59/Davtu2YbkeIBrcHj8JMWSg6fHdlfQUWR9ZunojifI/sr7KLaIHmG8PihoHXT9/C+8qajNBpXKRCTRuMqedNWUe1D1WUYYjDKHvJEeKi6zA77zdthv3lkOgYimJ//U0bbXkN94Vtr7huVa1JHtur/NaVUn2s5eZ8urK7UPZvZsuaR+yo53/6UNf808A2tvafJGum2fP8drHTzMbSSxLbDU6l7DRJEImCwwEN8wFUe8L33G4bBq7/akiigNWyP7MU2j6y5JO8rWP0AEY7wmHqb1+QIDdXWr/ex6QWcaJuprOXq8mrUg0/Azcvw4OOJAtzPfv2y/R5vrVmNPAez+1qw2tKAJdDciqsLgN1TLQw0iUjQMWSu7ZvTldrpmqu19zRZIN0QX9wWiQXAVpPQbBN8w1dptreoLsvj9ZJfoNScYsoopbRs2HdfwxBC5UXLbvBeWXOmUsQmZgmVKivbb3qBz8x+ib2ME6OSS9MLVFdYSRVlRXlMzC5SVmQtLywqGm78Mwpnx5gdDvLqomJset6zNmot5Fh2X9j1PJa0DJYB68UKv1+0G38mDNimdqXW2nuaLJFuksR3ROTvO39YGX2abYJv+Mojw80rk01G36FUTVkhNTWFjL7jeZ50Ew3MxUXe6vwRAr/9Ua4899cxFxetbL8iK4uvrHgpi88ryeLqnXHG7dT18ZkFrt4ZTztb0Dn/6A9uolZq/ZF9hlzPg0nLAAEgYof2urAMlkazo0jXgzqL1aQwai/r8N42It3wlW/TvuqjSFElzN6zHquPOi9YXsvk56kl7Rcbuc3h2TfIF5PDs28QG7nNYkn1MqMzOjHHvj3FJI9UgL0lebxQ8G8TCRV7S/6c2NQ8UzNzVDFObKYyUW+V8h5t43h49g2uFD3K0Y7vYuTlbeKnvWn0YmleRoAGLD1MN4P2eie0t+F5Ybd6hEaTC6TrQZ1XSj2jlPqanSTx8+s9oYg0iEjvel+vSQOP9HEnfLVW5XD7xXD6Ovziy9Bxw1r2SBV3hF3zhISwK6Zpbf+to4n9QjUPcLXoURaUwdWiRwnVpM5rOcuOcnmeCI/XWcrl+VN3eMK4SoHAE8ZV8qfuECzJ48Wi3+Dlon/K+eKvECzxNjpexjEXscN19fZcU8iVLNFrb+/G6tN2Ajjul+WXLs9fGuaL//V1QKtHaHKHdD2oFvtCibOUJHF8PSdUSkVERKuhZ4o1pI8n460IbpOXD/seWVr2ShUvrQYkoeKgFMjEHRh+GVBw4/swcQep3M/hX/kLrg/f4GhdHWIYVFcU8cTBPVwfHqau9mDC+1mysSqxHGRy2biDTBK7c5vHeJsCMXlMvU3szm2q709NDgjVPMCVokcT/aCO1bhuxum0D9lERKROKfWe33alVIf9tM+1rsVjew8bxJE2+urf+8iSeoROL9dkmXQN1DeBP3MtN2zwvLENvl7jR5oq416ICH/wj59Y1sbCF48swOjEHK8OR9lr3iNyQ9k9mQTHuICy0sFNxc/+537bEN6xQokoXij6DaT4MhQ9jqhvLwsbLioSXp1ZcogQBnnKZBGDaMkhJC+PQfNhO+z3MGGfrEQxDI52fJfYyG2O1TyAOMZ7A4Y9zc/2bwEopf6HvXwI+A7w8KadZIOkSBs5GXw6vVyTJdI1UD+llPq/nQURia6080bZtBTZ3cgGOtiapncbC088xGKryvL547LnODzn6lYrNVD78aXxlNdYhuzGCGHzNoM3TKKTc4RUHHP4EvkssjB8CWNiBKNyn53FZ1A4M8ZsUZBgaQEjH9zGwLTS35WJmhqh5v5aRosLkDmhoriA6gr/OiYjL4+q/Q8uX7kBw74adplGE3BIRH4euIvl9XSs+MItYsW5J53Bp8ki6Rqo6yJyHsvzEazuuusK8aXDpqbI5hDpdpHdED4CqemwkppCytg9xGLVxChH5t8kX0yOzL2JmhhFKvfBz3172XiqSg1eLWpfSlsvHebORCXXFx+mybjKgPkwh1Ql+4DY5CznzH9DQ9FVIuZhYpM/Qnx6AbfpiE8vsH9ihGPzbyCieGT+DWRiBCr3p/+5bbA1/SrElVJNIhLAysaLAo1O8Xu28VQu1+E9TQ6QroEaZOnCEqyiXc0a8M2QW8Pr0zVuyR1s08VvDspT6mhqNKV/U5Q9DJkP0yBXiaiHCbOHao/xqDtXl6WtqztXkdJD2J0FAYXYYUGv3lELVfdz2TySyOI7WHU/yETiNWKHEtfEBgx7GowCKKXiItKvlPox65Ty95VSf7SZJ1ovOrynyUXSMlBKqXO2Bl8TVo+or633hCLSAIRFpEEpFVnvcbYba9J6S5qsX4tx24gh9EtHd6SOgITUUag0xKvmw5byg/kwHysKUZVv8M/u+03iN9+g8sGP8GJFkWd792hpmCJVQgVWu4zZ0jDVEqc6720EeCLvbZC7QClSnto7Km98ls/O/2pCn+9yXp4VjksKJa4Zpy5s8zlr6+oJgIgs2s8VkDs57gPfWDJMP3hdh/c0WWctUkdxrLBbWET+lVLqN9dzQtsoZV1HZqtZMUPOjcdkfXRyPm3jtlHRUy81Ba8UcC8R16P7ynm+8KtQdAmKnkDUtxmdmOWXb/+yq737d6mqLOZz+y8uGbLKYmTCQC07h5E4WXLvqFBZIeVFhYzO7qGiOJ9QWaE1qKRQovVxbkFYdXValVJ/mLxSRH46G4Px5fUeyzDt/4j1p70nTZZJN8TX67rAXnIykjTpk7bWm8dkfVV5dXrGjZXDdOu9UVsGwZIbKi/KI1RWSFV5IeXFhYzOWEbi8L5ymLwDN79vuQbD34fJOwRNk5DxNgZwXN5GmWOI3M9/OfUJ3h354aVswfIaxPaAJMkDSg4RxqbmmZq31CWm5haXinKTPKCNhlU3C6XUH4rIY/bz19zrt3wwq7H/I/APv53tUWg0QPoGKuwySg1YDQz/R2aGtHNJS+ttg23Kvfbd6I06NjXP9JxlEKZdBuHVLy8XhjXN5a6xaVrirkFFIqA1Nr1AsNInW9DHA0oee7C0gLJCoXAmzmxh0FfWKFdaaIjIc1hZqVER6VFKPbvlg9BotiHpGqhu4FkswcprSqlfzNyQdjmb0KY8ed/o5NyGhFSrygtpqgumeGX5+QZH76tcOo8EGDKP0ihXGVSHCUuAqn2FXCn8IQ7PvcnVwkc4tu8Ao36Gw2MOyFPdQpkpmX3VlSWe407X88w0SqkggIhcyNogNJptRrpJEnft3jMBABH5eaXU1zM5sF3NWibr01A/cIRUx2cWlgmpLswvcH34BvV1dUt6dB7HS7eAN1iaz3uGQikTw1AES/Otwthn/r9lhbFrMRye+07cScnsg1QDlUMtNN51Pf+O8ySXsvg0mlwk3SSJ72D1ghrDUlY+BGgDlW3SVD+ITc0zZQuxTs0uEJuaZ29xHq9+5ZM8xtsMcoSPffl75OcZnsdLt4B3bOR9GtXbGAKN6m1iI+9Tdd9BEANVVrPM4HkZDq95Ms99PTL7/PBs9bH1iRPd9g88ALGf514Wn0aTY6wlSSKRWq77QeUIfuoHSV6QV4ju7XevLdOue/e996xw3fAroBatR/t46c7lhMqKEpNQItay3/xXsuFYaZ4sxciIIP/gj+CdXjj8E2uqWcpS4sT2yOLTaHKMdA3UcZeSBFj1UBlTktCkiZf6gaMefusSPPgkfOFbiGGkJk6UhBg0DyeKXStLQlBaCYVlMHvPeiy1uqqkG5KT8mpUUSVq9h4UVSLl1f7zTUmsKaFhYQ6+er9lSCUPvvQDyE9vfilLiRNtIjLm6PA55FIW39NTfwYfaOUITW6RtgcFDLiWd02BbU7jlVAx/mGKejiV+1O8kKP37+Gj8msUz95lujDA6/fvsY4zP2XtMD8FU1Er/dtnDiolVDYVRZJen26KfFV5IccfrEjURq2Y0HDzsmWcwHq8eRkOpXdjzVLiRF+ycQIQkUql1L2tGMBqfGL6z60nuvZJk0OsaKCcSVyl1LmkTa+KSB3ASu0CNFtAckKFh3q498sMXvu1n1iWJk5ZNRx43PLIDjye0KPzmoMCUuWPyqpRB56wvbcnkBVS5JONm5iLPB//HBTeg3glYl63Wnx4UfuU5Tk5HlTtU2l/XCslTmRwbqpFRLy6ADaTS5EIrRyhyTFWNFBKqT+yJY6CQL+9OohVB3XeXXSoyTA+2XopN9U1SP4kp4mj1HI1B6USLS8iN6IEzHsM2m00TFOlyB9VVxTxubkvcX3mBnWzdbygwJDUOSTPeaDRq1bHXrBChKNXl/efciMCDxy3DOGBx9esm+eXOJHhuSmvFjNjm3kCjWansWqITyn1Nbt3TTNWmnlEp5hvMT7Zer43VY+CVy8Dl2zczIkRz5YXVWX5/HHpVzk8/yZXCx6hquzH+eDe7LIhLiqrbcbA8F0WzD1Eh+MrzjclG7xQ6AjTUppQOC8JHVlq95w89qlRuD0AKHi/f1NaY2R4buo7XtJgItK9WSfQaHYi6dZBXQeSw3yarcInW8/3ppoc9vMwcCaSYtyiqoIiVUgF00xRyKyqoBpQ43c4tvAGhsCxhTdQ43e4O1W6bIh3p+Y5el9JevNNZfl8s/y5RFfbqrIfZ3RygU/MdlFn3mbIOMDL0wtUV+SBaaJ+91NLYcMvfCsjrTEyPDf1oyIS8UiSuLuZJ9FodhrpJklosonPDTntm6qHgYuqPSnGrcoYZ1FmEaCUWSqMcaCU2OQcIVuuSJTlbTxcU0O+mATUOGNSycM1VvLECz//OLGR24RqHliax0nygGQqyrGFK4iY1qOdTNFQG+L68ASNtcElDcGJEczhVyyv7sYrCa9us1tjZLKoVyn1o5t2MI1mF5FuoW6dToZYI2koPKS9n4/8kd9NdWHB9E5+uLVk4KogxbgJNeTVPpVSABsqL0KJXVkq1vLo1Bx/UPAVu/fTYcamnqa6vAjj93+SKncoElK9t5IqruQd5cjim7ydd5RjJVUYSvFC4W9AsaOE/i1r/svuMWXJJ/n3mNoM1iIntRXYLTp6gQal1FmffcJAp1KqdUsHp9FsAel6UF3AjzkLuZQem5OkqfCQqFly9vu5b3vvB/7yR8pEJu9A2QMgwsKCyce+0sv4zAIVxfm8+uUW8g0wlUJM65yilGdtFIB8YXkbd7DqmygoR81PIAXlSHk1VZOjqXJDU5IaioSU4t8PF8oYn5lHGYrxmXk+HJ/mvvxJ5FZqGNOSTwKlFIZhySnlikp5JhGRE1i6l30i0iAiJ5RSPR676rxwzY7F526YwnUROS8iv233hnopk4Pa9njNGXkxeQeGv2/fvK32FL6YplXTpJby7MzFRd7q/BECv/1Rrjz31zEXF3l3ZIKJmTmquMv4zDzvjkywcO8O5s3LCIss3rzMwj3rPAaKarm75IGYJur3fhL1O59E/e7fsc4JmON3LOMEqPkJzPE7SHkNRu0TKMknr9b2tpxQpJG/FIosDVlFv5Ao/h0f/YBG4x0KRNFovMP46AferwXGRj/gMXWVAlE8pq4yNvqBt4DszqOFpdrDiL28DNuIeRktjWZHoFu+ZwKfeqJUBIUgKPvRxwvw8chiI7c5PPsG+WJyePYNa7nmPi4U/4bV6VaOcLjme1z9UBFffNhWjXiYwFQJRys9EifGP0SGl/o5meMfYuy5j9jU/LKWGbGpeYIV8LOzXyQ++waVsx+xUsoNj1DkZDSl+PehQ4cY5EhijI2HDvmGMUM1D3Cl8BEOz73BO4WPcKzG8hRzRaU8g4Rdz2NJy05n6gje6euISBtWiw9qa2u9dtFocp4tb/kO6cXWtzWOlyNYN1rlXTBrllbzVsGjiVYUR0urvV1anyy+UM0DXCl6NJENd6zmAWRqlCbjHcQ0rcfpGIf3V/GxvF9j38xNPig+yF/ur/A8pmOInCHHpuap2gOhfQdSW2ZMzPDPP/hXVqfcDw4TnfgLq+VFcijSI8HDEOGHv/g/+cur12g49tCSkrpHGFOAo/dVIjcNjt1X6fRMzxWV8kwy5HoeTFoG6LTXBYFmEWlTSiXS1u3n3QBNTU0KjWYbsuUt39cQW9860k1oSJepUbh1GczF5SKuSUQnZxmfnUOJyfjsHNHJWesmnzwejyQHwGpl0fHdZa0srH2fQLn2NZTi1Qf/L7hldasVfgxKQqiCUkt3r6AUKQkRKhXeLHg0Ue/0yL4DS+dJapkRVHeXzUGJuotXywsvz8g0Ff/gdwdsD2hsxTkkv9osJzwJG08xz1F6sX4QRrCahF50b1RKtQCISAA45zZOGs1OId05qF6l1DNKqZds2aONaPGtGlvfUpzw2W8dsxIW7HmXDeEzn2IuLjL6g5so+xxVxDluXKVA4LhxlSri3uNxPDKF9dw1D2Xk5VG1/0HLOAGmgs/NPsvfnvkNPjPzJUxl3+RvXsJQiywOX7KWJ0dZnLXmlRZnJzAnR1EI/zb0NT4x///w61W/iXKHHJNaZsQkwKD5MPPKYNB8mJgE/D9ihBG1J6FSsZY5pCh7lp0nyp7M/M9yDPtHW72INAMhpVQfgIj0ZndkGs3WkY2W7yvG1mGL4+d+LSs2gpfXYCc0HJ59gytFj3K047sYYuBM7AiCiOGfYHHLFki9dXn5GJO8rahH6A2PAlxTCdcXl9TMD5mVGJNzDN60lCBiLiUIr6y5qooi/vf7fovrN29QV1vHeZ/0bM/XrqVhYUURv3T/b3F92DrPixVFMDmy+f+zHEQp1WE/7XOta0naJw7oFHPNjiQbLd9Xi61vbfx8o6oEfuHBpPkUr4SGqn0HkNqn7NDbk0v7e7TQSA7HJc6dlOhQJfdS0r9NpVAykyjALeMeUSPAZ+e/TIhxRqnksmH1jWqo3cvg8BgNLsPh5fGEygpRYtUi1eE7zeardpHuHJKI8Hzbx4lONi3tmwElCY1Gk3uka6CUUuoZsBTON1gHtWJsfcvxyR5Li3TrncAzocE695+m1B2Zn//TZWoMC+MjMDtBPrAwOwHjI+QH9sPUKOrmJcRcsB6nRhGPbrOxe9MUqSIqmLE8KCqprijieF2IwRsGjx/cS3VFkR05VIkwomN0vDye0Yk5IsNxFhVEVtDd8/OW1lIUmzLftJH/mSaFp6f+jEfnXgd0LyhNbpGugeoUkQ4sPT4nc2hdgrFKqR4R6UyOrWcVvyLY1fAwEH7H8UxoME34vZ9cVqhrInz265ftG/otXjj1JO9OlRB3heMCUyUcDYBZHGRaFVKqFphShZQUBzFEkM//sdUjqfYpq6Ou3E14UOXMUCF3ESnlxbanlnkxoxOznkbHS7Ei7SaGG8248/sRsN7/mSYF3QtKk6uka6D6gHZgr1LqWRE5tZGTesXWtwPm4uKSZ2MYmCVVvJV/zPKK8o9xtKRqKetkcWHJM7K9KiehIYFTqAuJQt0oe1NCYof3V/BY3r+hcGaM2eKglSYOxEY/IGBOIwJF5jSx0Q+oqr4PdTYMc+NQWIF0vIeopSQCgcRysmeyktFJ9nh8dfc82FDGXZpt7TUb443Cj/Co7gWlyTHSNVARIKiU+lG79cauwyvJITq1wKcnnmGPeY/4/B5enpq3buKLC3D2kDVnVFQJp/2a7wmJ6lf7uZeREBEiX/6x5fp6OGHDH1oWNjR/8AYyN24ddW4c88MrGIaRyMcTgOk4VNyXohLuJ3/k/YGYqbp7PnJOKWrkfnJOXngVPa8htKrRaLYv6RqoqKur7seA8xkaT87il+TQWLuX68P3aKx1eRx28z3AevRrvldeg6p9KnGjFbu9upeRSGkuiHfY8M5iGTUu1YeRxTL27a+DwoqEV0XNMbu+6GXyMVm48fJSfVG6c0M+nk2yUK2vGnm6eBU9a69Ko9kVrGcO6jobmIParixJ7thqCjUPID4K3FQdXt6SvOqw5zFNBZ+b+zLXZ29QN7fUgXYtJIcNx2YWqWbJLxubWWSfYUDHe8tCjlbvpyI79bwo0fspbTwy6byEasdsNXJL9XxJjTxtvIqevbL4tFel0ew41jMHdXKjc1DbEQGO7a+EW7IkuTM1itxcrtRNeQ1Mx+xf+1iP0zEfJYnUDrShskJPpe6U1u4+7A1at3/H8XCWyctf5sVZvZ/m7NTzuUTvp/Q/kNRMundHxhmfWQBgfGaBd0cmOLK/IrWOaS14GSOvLL5dUhul0ewm1jIHtXc3z0ExNYq8n1Qs6yh1z95LKHUDadfp+KZvJ7dD9zFaQEpYq2rsfyWiYSioGvtfEPrrKef2SkdfM0mZdIf3lVNZbCSSOQ7vK/euY1oL6aaU69oojWbHka6BagaeYymsdy0zw8kS6cxdeE3WT46kKHVTbskBJdcyeSEi/ME/Os714RvU19VZ6dtl+fxx2XMcnnO3Q59j4MYYi6ZiwN3a3ekndesSPPgkfOFbjAYbCWGQp0wWMYgGG/Gc8RHx7P20kc/NAF6r/Q/Izcvw4OMIViPZDTcCTKOFPYaha6M0mh1GukH6RuAZAKXUdXaStEq6um5KWVJ49h9K+WvumYrPfv0yT/6H1/nMuUuYplo6l6unk7m4yNWv/Q0O/X4Tb3X+CObiImpilCPzb1IgJkfm3kRNjBIozsdQ8xzmJqhFAsX274qJO6jhl8FcRN34vrWcZ3Bk9j9zcvbLPDT7e6g8I3Eutw4gpgm//1PQ9des97+anl1yPyqvz21qFOPWZUQtII6Xme7x1nJuPzkox5Bp45Q2z18aToRlNZpcI10DNYh9XxaRnybbAq+biV1si11s69zskm/o5sQIi8OXEJfgKiLw+T+B9v9p/Xq3b4zRyTkG3xslYMYZeC9qiaF63NC9MgO9xFGv3bnLYGE7/29RB68WnuLanbsALCwqlC3BqlAsLFryrosUcplHgDzEfi9XnvsR9vz2R3nTbmzod5M3TcXI+CzKbTh8jFHK630MdgprEXv12jfd82hW5ZuvvQ9AVXnutLrXaBzSDfFdAJ4Vke8AY2xnA5UUlvIqtsWr5skrG83xQpJCTcGSPF4oWmoaGCz5ceucSa3P/eSPkpMK9o6/Qx5WQW6FmuYh3geCvDtdSnzxyJLCxHQpwfLl3oOIEP3wfQ7PvUGBbQijH75P9b4HwKXvR0nIv5X6SsYoOXnh82mEDdci0Ou3rw7nbRoVxfnsqyjO9jA0mhTWo8X301iddbcfHnMX0al5fmq8g4AaZ2yuklem5pHJO541T+mqalttyt+mQEweU28zNvoBVTX3W4Zgbtx6LA15yx9BSlJBtKyeEkqoUNOMU8J0WT37gHBVGUfmfzUh+Hq1qoyCgjwer63kveGbHKytpbqiiBFVzaC5JJUULquG6SjMTVqfy9wkTEeJqj2ewq6UVaMesPtRHXgc8cuk8zHYKZSEUoyjL37JD1rqaFPQOnyaXGa9Wnx72Y51UB7aecHSKkqKChmd2UNFUT7B0gKMMo9OtV7ZaD43Wrdn9I7jGU3esYwTWI+TI1CxP1X+iFRpIBF40zxAo/EOb5oHCNsOw1B0CoXBKHsSy4dryvjXY89yuOgNrsYfRZnfpbqymF+6///g+vAwdbUHOV9ZDBRD7ZPLbvxV4Klmbpomb/3gHodNxdUP7nHUNK0uuMlGIl3PyMM4+hqbdL0yzbrQOnyaXCYrWnzZwktcNTY1z9TcIgBTc4vEbLkiL88mJRttOoqam7RlhSYR+0br7Rmlyhp5D9JE/eefsL2VJ5B/9N+olrtU5b2DATye9w4id4FSK627yKBwdimtO3bnfY7MWd7fEcf72/8gf3Dq47w78tFE6jeQ4gEp004BSVIzj43c5vDcm5ZHOfdm4pgp+HT+9dwvyTj6/9PS9Mo060br8GlylXSv9AgQ3+51ULHRDyhKFlctL6Tp4F7yDaHJ3Q4iqVMtkJqFV1LFlYJHWFAGVwoewSypSuya8vryGktd3MiDgx/39RjMu+/DrVcQFNx6BfPu+4gYiG3QEo0NAZTihcKv8HLRP+XFgn8HShGqeYCrhY+woISrhY8QqnkA01T87H+6xKf+4/f4rDurMCnrLTo5x6CtZj5oFw6D5RFeLXqUBWVwtehRQjUPeH/ASqX+eeGEB//FFfi5b6/sFfll7Gk0mh3PrtLi8xJXFRH+4B8/kdCP8y0kdWqOXK0xolPz/N3xf8lj6l0G547wiiMW64WIdTNeZWJ/bPgtgi4tvbHhtwh9tNmzsWFs5DZHbM/miOPZ1DzA0f0VyC0joXgx6tM0MDlhJFhaQGlhHuMzC5QW5hEsLbCGbhgcPf3njA3/FccOfnS50XYzcQduvmINfPhla7lyv/e+6c4h6QJcjWbXsqu0+LxCb4534anS4MajNUawOMhA4T+hginGKaWs+MbKA0jjprwYfCh1WQQ+/81lPZ7AuwmiOf4hcvNly74Nfx9z/EOqKventtHwUBmPTc0zNWvVxEzNLiTCnZgmxn/5NKHVwmyOfAVYj5sxX6SbE2o0u5adq8Xnow6RnJQQ9VNpSCF1Dmnsxl8RYspO/54ieuOvqHqowff8aenpVe7nkvkwjcY7DJoPE67cb7Xv+Fp9SvsOL4MbnZgh5PLAYhMzVO1JVUg3x++kqIxXVdTQVBdM7QeVbvJDeQ3UfnzJ29msLDudsbermZ+f59atW8zMzGR7KJp1UlxczIEDBygoKFjT69ajxVfHujrPbSEeEkB+E+vB0gJKCgwmZhcpKTASYa0UPFpjmIsVjLvSv82qo77nNxF/PT0XVsbdv1+ecXfnCmr2nmVzZu8hrvYdyQY3JOO4mz+FxMocTE7w8Kzr8ut+m26YLc0wpiY9RKQT6AUalFJnk7YFsCIazUC3qwnojuPWrVtUVFRQZ8uBabYXSimi0Si3bt3i0KG1pTCka6AagB8VkRas298h4KtrG+YSItIAdCqlMlPwO3HHmgNBgS0B5MyFJHsxo5NzTMxaWXwTs4uMTs6xr7I4taBXwWdnl1pjvKggVFHMx/g6+2dvcbuolr90ih1tCSJBoW58H5m4Q1RSO+V6eWpKgRKDKHuoE7GWQ0eYllJKzSmmjFJKQkd8s1ukvAaVtOxFVUUR/+y+3yR+8w0qH/xIQmXcUzfPL8ym+y9lDBE5AVxTSvWJSIOInFBK9bh2aVZKtdr7jolIl1JqKDujzSwzMzPaOG1jRIRQKMTIyMiaX5uugQoDZ7B+rfXYj+tGKRURkaaNHGNFfOZCTFPxs13fS9yUX2j/OAIYLPAQH3CVByznw6Ogd2R8jss3YoSAy+/FGBmfxTCEqXmIUcn0vErM2VhJcsqOslnp2lUV/u3U3UQn54jYmXQRVybdJ2a7qDNvM2Qc4OXpBaor8jxfv1BSzWsc5TF1lVflCB8rqSafVMMsSvF80Rmrl1Wxq5eVH+kItsLGejJpg+emBeiyn0ew9C8TBirJWA3sVOPkoI3T9ma9/790DVQcS+IIrCy+E1jhhY0Q2+Dr/fGZC4mOT9H1g1YqCqYZ/0EJ0fFrVJcX8nrJLyS8k9KyYc+CXqGcFwq+klBjEP4WVWVF/EnZGUt5vNBSHgeISoAh84gdPjtMWAK+4bNkw+HVggOgoTbE9eEJGmuDvsYN4N3RSVpnvpxQl/jvo5Mc3leRGl6cGkVuraF/UrLx8Ev/TlPCKGU+TjccTCbseh5LWk5gh/oiHuvbgDaA2traDAxPs1OJx+MMDAzQ1NREIBDI6ljWosW3Vyn1NRH5FaA7g2Pa+MXlMxdSNf0eyDQCVDBNxfR7qCkoVVaiQ6maQt25iqo5xlt5Rzm8+CZX845ytKSK6qkRgnnvkI9JU9475BnjyOQ9js3/FSJYj7Y6RFVFEb90//+Z0qQvWSHCNBWf6X6ZwRtxGg8GeLHtKQwjtQUHpskLhV+BoktQ+ASivu0bZnuougxDDEbVHvIEHqous+qbksOL5T7zSl5ejJfx8Go/AmnNVXlq/q1Fn2934PaIgknLbtq85p+UUt3Y12lTU1MakvEaP3p6emhtbaW3t5fmZit41N7eTn19PeFwmNbWVjo7OwkEAly7do36+nra2to8j9Xe3k5XV5fntlwhEAjQ0dHB4OBgtoeSnoFSSt0F7trPv7ba/nb8PJi0ekgp1Zfm+TZ+cXlkfknNMVRRJcrOhpOaY4zem2avMsjD6p80VlIHk7OMz86jRDE+O090cpZQyV5rTkmBiEIV70VmxvBSh/CURfK4yd+5N0v/e1FCjHP5PZM792aoqSjk6tf+Bodn3+AtW6jWmBwBZ07LXV/kUZs1Nr3gDm4yNr3g7ZX5ael5eTFexqPULkp2uiMqBYaRVh+sNRnM3Usv0ITlHTUAF5N3EJFm7OtERAJKqfhWDnC3cOLECZqbmxkaWvqNEAgEOH36dGLZ/dy9n5uhoSEuXLhAa2trwtDlKsFg8u07O6TrQa2JpPh4VvBM6TYM5PT1JV03wyAo4yhRicS3oIxjiLDXeJd8FI3yLnlyj+iN9wkpExEwlEn0xl8RPPQYbxU8yuG5N7la+AhHS6sTyQvJ3pLXTX5ssmBZ2HBs8hPkz4ymCNUGivIxXHNai4vK+sd51GZVle/jeFKquPhl56WrpeeVxTc5YnUWNhcT+5ql1Xz265ftc99aylRM7vqbrsHcxSilekSk0zZCIefHnYj0KqVa7B+BncCQiASx5oizft1lml//0zd48/a9TT3mI/dX8q9/8tEV9+no6KCjoyPhGYVC3gLH3d3dvt5TT08PnZ2ddHV1JQzU0NAQkUiE3t5eOjs7icViy5YBLly4wODgII2NjTQ3N6+6f3d3N+FwmN7eXrq6uuju7mZwcJChoSG6uroIhz2jxUQiETo6OojFYjljQDNioFbDzuILi0iDUiolfr5RlkJn1g3QCZ0BkJefSNEGiHnMF4VKC3m74JhteI5xrLQasyo1pTw6Nc/fneywkhfmD/Cyq7DVMySWdJM/Yn6IabxDvpg0Ge9gVMwi5anFt2//YDylrcbRvYAliOTy4cTXGKUYTC/80sm9jIfHvtEJD8+orCDlsxDDSM9g7nJcobs+17oW+7GHXWCQcoXm5mbi8TiRSISBgQFOnFgurtvd3U08Hqe/v9/XQAG0tbXR0dFBPB4nEAjQ19dHPB5PhAgvXLiwbLmjo4P6+noaGxvp6Oigs7Nz1f1bWlpobm6mv7+fs2fP0tDQQG9vL729vb7jisfjtLa2Mjg4yJkzZzh58uSmfXYbISsGyjZKGfuJPDLhDp0pRiZmrdRxD6z5ouVtNEYn5vj0xDPLWnBU7ynhMzUXuHvrTSoe/CEu7CkBZfLH5WcTxqSq7Cesg3okWVBekxr+Kqvh7UKXISzzFpo9vL+Cx/J+ncIZSxT2L/dXALBQFGSaUsrVFONSSmlRkHw8UsXTrQtbSTk82Xh4GC1Pz8inJcmG28BrdiWreTqZpKOjgzNnzhAOh1OMkLPshPccryUQCNDZ2UlfXx/RaDTh3XR3d3P69Gna2tpoaWmhv7+fixcvpixHIpGEZ+ScY6X93eHFlpaWhLe2WsjuzJkzdHR0EAgEGBoaoqGhgb6+PiKRCNFolJMnT9LV1UVXVxc9PT309/fT2dlJa2srkUgkMa/m7N/e3u7rqa2FrBiojGMuLgudYf5N311FhOdPPUlspDZhOIKlBSktOJQClZfPOzxIo2FYUy5TUY4tXEHEtB6nLDXzhaIQry4+zGO8zWuLD/OxohCG3QbeHf4amZzjU+PPWBl3s5VccmqwxECVLYm4GoZB5Ms/ltALNGzjcv3mMHXKEr8tUdNcvznMw/X1qW9yhbqw5Z/bGpXDk4yWp/e2Bi29tJQ2NJos4Xg/5875JzCHw2F6enpSDJjb0DQ1NfH0009z+vRphoaG6O3tpaOjg56enoS34ywHAgF6eno4ceJEwvistH84HCYSiSTmzFpa0is17evr49lnn2VoaIh4PA5YxjYcDnPt2rVENl93dzfBYJD6+nrOnj1LS0sLLS0tNDU10dfXlzBKm2GcYIcaKJmM0mhcpUBMGo2rjE1GIVBmbUzOUjNN5Pc+taQz93Pf9mzBAfDqcIy96h6RYWuiP1RaxRVXtt+xkioM4OrIBD8z+6VEqvefjUxQXVGcEv6yQnNL/ZwEnww3Q8jPNzh6X+Wy91lfV8egHE107m2sq/P5QNLUyNuETLoUzyjNuSXfbr4aTQ7R1ta2LLzX02NFWZ0bd39/P/F4fNk+3d3dXLt2LbHs3Lyd8B1AfX09zc3NXLhwYdlyQ0MDLS0t9Pb20tramjBSfvufOHGC9vb2ROjw9OnTnD17loGBgWVhxc7OzmUhv+bmZrq7u2losKTahoaGGBwcpKuri97eXsLhMIFAgN7eXs6dO0cgEKC1tZWLFy8mPC9nHuzMmTOb9nnvSANFeVIH2XJXCnXSfIg5/iEy/P0UcdXjtXsSYb+q8kJQJt8sf84VzvtxPrw3u8wDemV8lv2BEgLF+csMT6A437e+6fG6IIPD1rpqO7zopTjh5V0YeXl87Evf4/rwDRrr6qwmgp6fh3ddWMoxM6Ucnsbckmdmnw4BanIMxwtyOHHiBMrVViZ5bgpI8aYCgQBjY2Mp+/nt6zZuq+0PJMJtTqLD6dOnl2UZOvNpbtzvy3md4zXV19cnDOPx48fp6+ujoaGBYDBIJBIhHo8zNDREIBBIvMYxhhtlRxoo7w6yeHoIsal5gmopUzo2NU9VheL5wq8gRZeh8HGr7sgjnDc2VbTMEI1NzbE/UMLdmYVl47k7s8D9Pm09Xmx7ctVC3ZW8i/yC/JSwXorh8agL8z1mljLp/Ay4RqPZfLwMaTKO0XKMW7Jxdgyh8+hsT95vI+xIAyUivND+CaKTx5fPZ3i0aA+VClcKfyihBnFs3wHMiRHM4UuW0vfwJYyJEYyKGsT2LsTJwitVVBQJRbNxZor2csROXjiyv4LyojwmZhcpL8rjyP4K37YeySExr3mc0YnZtL0LX8OT5MXkmsfimwoPWgJJo9ml7EgDBT5p1R4t2qW8hsOnv8v14RscratDDMsjGjIfslPPH0oofSd7F4ZS/OXB/4jcvAwPPo7wY9a5DYPXfvVHlyU1jIynb2SSjdZavIt0DY/nMbMsN+SZ2aclkDSaXcvONFA+NzWzpIq3Ch6x5pEKH+FoSRWYiv/tP12y55s+5IW2p6gqL2CkqACZEyqKCqgqt1tweBS2Grcug1qwilZdSQXJSQ0bCWGt6F0kke55PI/pkxKeVbQEkkaza9mZBsrnphadmufTE8+wx7xHfH4PL0/NgzL55dv/goaCq0RuHyY68RdUG+McW3zLmm9afCuRPp4yt+MRMvRjLUbGi3TrhtZynpRj5mJ79Vwck0aj2RJ2poHyualVlRfysdoQg8PGkncxcYe9hiUC2yjvkCf3oCx1vslzbscjZLjSr/utKk5d93lyUW4oF8ek0Wi2hJ1poHxuasru02RV3VrCr0ZpFXlFZajZe+QVlSGlVZ6vj3okKoRKU0OG2352JJtyQ37JEFoCSaPZlexMAwWeNzWvZoDVcheZm7R2mJsExwtKer3X3I4jieQOGeranXWikyE0Gk0SO9dAeVBVXkhDbYDBG2M01AbsBIJqqH1y1TkOr7mdqvJCGg6GGLxhpCQkaNmeNaKTITQ5TCQS4fz58xw/fjyhVecUtDpadI2NjYBVF/TSSy/5Fqpuh55QucKuMlBLBd9LIT9ZQ3FqOjVLoGV71oVOhtCkw397Bn7w+uYec/9H4Cee893sKH27FR0aGxuXSf4kqz3EYjFPA7WdekLlArvKQI2Mz3L5PUti5PJ7Y4yMz7JvT/GG5ji8EhJyrQh2W6CTITQ5Snd3d4oxOXnyZELZPFmVYaV2G7neEyrX2FUGKvmel6l7oJbtWSc6GUKzGit4Opmiv78/5YYeDoc5f/488Xg8bcVwh1ztCZWL7KpZ6OqKIh6v20uewON1ezPm1Tihv5effZoX257Uc1AazTYmHA6niKs6rSicdhRedHd3097eTkeH1XfSqycUWAart7eXU6dOeS5HIhHa2tpoa2tjbGxs1f2Te0L19/cTDAZzpo37WthVBkpEeLHtKV75YjPn25/aFMNhmoqR8dllisawFPrTxkmj2d447Svc9Pb20t7eTnt7e8KDcYjH48Tjcdra2ujq6kqE3ZyeUG1tbZw7dy7RlsLpCeX0kkpednpCOfuutr/TE8rZf60eXi6x5SE+EQkA54BmoNvV1npL2MxiWZ0ModHsfMLhcKLvUX19PfF4nPb29sQc0sWLFzl16hTHjx9P9E1KnrPKtZ5Q2wVJ/uWf8ROKnFBK9djPx4BGpdSQ3/5NTU1qYGBgy8a3FkbGZ3nqzEssmIp8Q3j52ad1MsQuQUQGlVJNGT5HJ9ALNCilzq51u8NK19AbX/0kAI9+8XubMeSMcOXKFY4dO5btYWg2iNf/cbXraMtDfI5xshlYyTjlGsnhPKuuai95htCgkyE0m4iInACuKaX6XMtpb9dodgJZm4OyQ30Rn21tIjIgIgMjIyNbOzAfnHDeU2de4jPdr2CaylM6SaPZJFoAx+2J2Mtr2a7RbHsyMgdl/5pLThkZcn7t2bT5zT8ppbqBbrDCE5kY41rxqm0CUqWTdIhPszm485pjScurbheRNqANoLa21vck44HtETpTSumEo23MeqeSMmKgksJ4KYhIM7YBEpGAUiqeiXFsJn61TbreSZMh3KHvYNLyqtvT/ZH35D85t7FRbgHFxcVEo1FCoZA2UtsQpRTRaJTi4uI1vzYbWXwngE5gSESCwBlgRYOWC/jJGm2kx5NGswK9QBNW+K4BuLjG7TuGAwcOcOvWLXIl3K9ZO8XFxRw4cGDNr9tyA2V7VzlvkLzwSlHfqh5Pmt2FUqpHRDrtaEPIlQzRq5Rq8du+EykoKODQoUPZHoYmC+wqqSONZjvhmqPtc61rWWm7RrOT2FVKEhqNRqPZPmgDpdFoNJqcZMuVJNaKiIwAN5JWVwGjWRhOptDvJ7fxej8HlVLbommVzzXkJlf/X3pc6ZOLY4LVx7XidZTzBsoLERnItMzMVqLfT26z095PMrn6/vS40icXxwQbH5cO8Wk0Go0mJ9EGSqPRaDQ5yXY1UN3ZHsAmo99PbrPT3k8yufr+9LjSJxfHBBsc17acg9JoNBrNzme7elAajUaj2eFoA6XRaDSanGTbGShHf0xETmd7LBtBRBpEpDfb49gsRCQgIhdFZMzu9LrtEZHT9nva7t+1Fa8Zr+1bcZ2tdA6/75P9Pxm0/wJbPS6/MWT681rls2qzx9IrItec5pWZ/qxWu4dtxvdqWxmondRFVCkVwVKj3ik0K6ValVJ7gTYRSe5ftK0QkQal1FmlVCtwMtvjWS/r6cy7FddZGufw+z6FlFKN9l88C+NKGUOmP680jt9nj6UFS4jb0WbM6Ge10j1ss75X28pAsfO6iMayPYDNIqkH2IBSKrl/0bbCvvgQkQagK8vD2Qjr6cy7FdfZiufw+j7ZRqpBRFQGf5yuOC6fMWT681rts3JfawHbaG7FZwX+97BN+V5tNwO1WpdRTZaxQwmRbI9jM7Dfy7OAZ+fnbcJ6OvNuxXWW1jnc3yel1JDtJTQC5zIU4ltxXD5jyPTnle5n1QAMrjDOrWRTvlfbzUCt1mVUk33aXG0gtjVKqbgd4ovYF/92ZD2debfiOkv3HCnfJ9u7vUBmDGda40oaQ6Y/r3SPf9IeU4IMf1YrsSnfq+1moJwuorDDu4huR+zmed3280B2R7OpJF9c24nVrhmv7Vtxna16jtW+T04YdqvH5TGGTH9e6R4/7DfXlKHPaiU25Xu1rQyUHZeu3wldRO1f5OFt/Mt8GXacuwu4KCKDQHOWh7Qh7EndLvt99WZiknkr8LtmnOwrr+1bcZ2tNi6v75OdrdZlvyYj84JpjCtlDJn+vFYbk/08DPS7ljP+WXndwzb7e6WVJDQajUaTk2wrD0qj0Wg0uwdtoDQajUaTk2gDpdFoNJqcRBsojUaj0eQk+dkegMZCRALbNVNMo9kp2NmDF7GKs+NAPXANq7D0ItBur0Mp1ZG0fwhLyaF960e+M9FZfDmALZwYytUCV7sGpVNfeJrdgIgopZS4lsO21FJivYiMAU8rpSJJ6weBju1cApNL6BBfbpBz0kC2UvFpSCgqaOOk2XWISJuPrmSKBp2H7JFmg+gQX+4Qtg1Ci1KqxfGqgDNY+lotWNXX7VgV2Sexwgot2GEF+wL5GSCApXwQtLf3O8d1n1BE2rC0usLArwCPu14bBo7bBYBhoNU+R5vXMV3y+ceBLv0LUrOdsb/nAazvc3fS+hagx63OYBeoNgGn9Hd/89AeVO4QU0qdBRwlYkcgM86Sh+WsO4slqR+ww4KOakPQXt+HZcAGXPs7x3UzBARtI3Mv6bURYMj+9TjAkkRJyjHt44bsdQ36AtVsd5RS3fb3OVkHsNtuA5K8vgXLq4pv3Sh3PtqDyh3i9mMM65fbavtFk9fZcfLkRmAx12PycWPOdp/XYm+Li0jy69zHHMKS9g9g9aPRaHYEa2wb0wIMikjjdm83kytoD2p7EGBlowUkwmxDeBuJ4Hpfu4LwaxASXl4XllhlTiZ6aDTpIEvdaL0aO6643jZKHUDvdm/YmStoDyo3aGDJA2kCBpVS3WK1Rz6N5SGdxJr3Cdv7HccSXuxjSbBxCGuO6iLWvNHfBppck7dNLE/IaHZtT37tKJZApxM+DLvGt+yYIhIHOoEhERnCymKKb97Ho9FsDbagqax3vVKqG9eclWZj6DRzzYaxf0VGsEJ+Qax5KB3q02g0G0J7UJrNIIjtQWEVNV5YeXeNRqNZHe1BaTQajSYn0UkSGo1Go8lJtIHSaDQaTU6iDZRGo9FochJtoDQajUaTk2gDpdFoNJqc5P8HsgTliXLtcsUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x216 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn import metrics\n",
    "f, (ax1, ax2) = plt.subplots(1,2)\n",
    "median = np.median(results_dict[\"ground_truth\"])\n",
    "ground_truth_classification = results_dict[\"ground_truth\"] > median # Classification: larger or smaller than median\n",
    "target1 = \"ve_dmean_weigh\"\n",
    "target2 = \"bam_oc\"\n",
    "ve_dmean = (results_dict[target1]-results_dict[target1].mean())/results_dict[target1].std()\n",
    "bam_oc = (results_dict[target2]-results_dict[target2].mean())/results_dict[target2].std()\n",
    "\n",
    "ax1.scatter(results_dict[\"ground_truth\"], ve_dmean, 5, label=r\"VE-Assessor, $d_{mean}$\")\n",
    "ax1.scatter(results_dict[\"ground_truth\"], bam_oc, 5, label=r\"BAM-Assessor, (OC)\")\n",
    "\n",
    "fprvi, tprvi, _ = metrics.roc_curve(ground_truth_classification, ve_dmean)\n",
    "fprvs, tprvs, _ = metrics.roc_curve(ground_truth_classification, bam_oc)\n",
    "\n",
    "ax2.plot(fprvi, tprvi, label=r\"VE-Assessor, $d_{mean}$\")\n",
    "ax2.plot(fprvs, tprvs, label=r\"OC-Assessor\")\n",
    "ax1.set_title(\"scatter plot\")\n",
    "ax1.set_xlabel(\"human rating\")\n",
    "ax1.set_ylabel(\"assessor (normalized)\")\n",
    "ax2.set_title(\"ROC curves\")\n",
    "ax2.set_xlabel(\"FPR\")\n",
    "ax2.set_ylabel(\"TPR\")\n",
    "ax2.legend()\n",
    "ax1.legend()\n",
    "ax1.set_xlim([-0.1,3.1])\n",
    "plt.gcf().set_size_inches(6,3)\n",
    "plt.tight_layout()\n",
    "plt.gcf().savefig(\"data_scatter.pdf\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Run crossvalidation to compute covariance matrices and models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cross_val_covariance_matrix(results_matrix, n=5):\n",
    "    \"\"\" Results matrix: Output of previous processing step. The last column contains the fold indices in range [0 ... 9]\n",
    "        n: number of Validation folds. n = 2,5,10\n",
    "    \"\"\"\n",
    "    corr_mat = np.zeros((results_matrix.shape[1]-1, results_matrix.shape[1]-1, n))\n",
    "    for i in range(n):\n",
    "        indices = ((results_matrix[:,-1].astype(int) // (10/n)) == i)\n",
    "        results_matrix_test = results_matrix[indices,:-1]\n",
    "        corr_mat[:,:,i] = np.corrcoef(results_matrix_test.T)\n",
    "    return corr_mat.mean(axis=2), corr_mat.std(axis=2)/math.sqrt(n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(8, 8)\n",
      "Correlations\n",
      "Correlation btw. ground_truth / ve_dmin 0.562 (+-0.0342)\n",
      "Correlation btw. ground_truth / ve_dmean 0.755 (+-0.0195)\n",
      "Correlation btw. ground_truth / ve_dmin_weigh 0.719 (+-0.0173)\n",
      "Correlation btw. ground_truth / ve_dmean_weigh 0.83 (+-0.0117)\n",
      "Correlation btw. ground_truth / bam_jvl 0.392 (+-0.0234)\n",
      "Correlation btw. ground_truth / bam_jvl_crop 0.365 (+-0.0242)\n",
      "Correlation btw. ground_truth / bam_oc 0.696 (+-0.02)\n",
      "Correlation btw. ground_truth / ground_truth 1.0 (+-3.14e-17)\n"
     ]
    }
   ],
   "source": [
    "# Assemble results to matrix\n",
    "results_matrix = numpy.stack([v for v in results_dict.values()]).T\n",
    "corr, stderr = cross_val_covariance_matrix(results_matrix, n=5)\n",
    "print(corr.shape)\n",
    "print(\"Correlations\")\n",
    "names = [v for v in results_dict.keys()]\n",
    "compare_idx = 7 # ground truth\n",
    "for clm in range(len(corr)):\n",
    "    print(\"Correlation btw.\", names[compare_idx], \"/\",names[clm], f\"{corr[compare_idx, clm]:.03} (+-{stderr[compare_idx, clm]:.03})\" )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Run the Hybrid assessors (HY)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For the combined models, we compute the correlation and the accuracy that could be optained by a Logistic regression classifier, that only considers the two-class problem of predicting if a concept is rated above or below the median.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To speed up computation, we compute the scores directly from the data matrix obtained in the last step. However, we also have implemented a class ``HybridAssessor`` that could be used."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LinearRegression, LogisticRegression\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "def cross_combined_correlation(input_matrix, n=5):\n",
    "    \"\"\" Fit a Linear Regression to :-2 columns in results matrix, to predict results_matrix column -2 (contains the ground truth)\n",
    "        The very last column (-1) of results matrix is supposed to contain the folds, the previous one (-2) the ground truth\n",
    "        n = number of folds for crossvalidation.\n",
    "    \"\"\"\n",
    "    corrs = np.zeros(n)\n",
    "    for i in range(n):\n",
    "        test_indices = ((input_matrix[:,-1].astype(int) // (10/n)) == i)\n",
    "        train_indices = ~test_indices\n",
    "        results_matrix_train = input_matrix[train_indices,:]\n",
    "        results_matrix_test = input_matrix[test_indices,:]\n",
    "        reg = LinearRegression(fit_intercept=True).fit(results_matrix_train[:,:-2], results_matrix_train[:,-2])\n",
    "        pred = reg.predict(results_matrix_test[:,:-2])\n",
    "        corrs[i] =  np.corrcoef(pred.flatten(), results_matrix_test[:,-2].flatten())[0,1]\n",
    "        #corrs[i] = np.sum(reg.predict(results_matrix_test[:,1:])==labels_test)/len(labels_test)\n",
    "    #print(corrs)\n",
    "    return corrs.mean(), corrs.std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LinearRegression, LogisticRegression\n",
    "from sklearn.ensemble import ExtraTreesRegressor\n",
    "def cross_combined_accuracy(input_matrix, n=5):\n",
    "    \"\"\" Fit a model to :-2 columns in results matrix, to predict results_matrix -1 (contains the ground truth)\n",
    "        The very last column (-1) of results matrix is supposed to contain the folds, the previous one (-2) the ground truth\n",
    "        n = number of folds for crossvalidation.\n",
    "    \"\"\"\n",
    "    input_normalized = (input_matrix[:,:-1]-input_matrix[:,:-1].mean(axis=0))/input_matrix[:,:-1].std(axis=0)\n",
    "    median = np.median(input_matrix[:, -2])\n",
    "    labels = input_matrix[:, -2] > median\n",
    "    \n",
    "    corrs = np.zeros(n)\n",
    "    for i in range(n):\n",
    "        test_indices = ((input_matrix[:,-1].astype(int) // (10/n)) == i)\n",
    "        train_indices = ~test_indices\n",
    "        results_matrix_train = input_normalized[train_indices,:]\n",
    "        results_matrix_test = input_normalized[test_indices,:]\n",
    "        labels_train = labels[train_indices]\n",
    "        labels_test = labels[test_indices]\n",
    "        \n",
    "        #print(results_matrix_train.shape, results_matrix_test.shape, len(labels_test))\n",
    "        reg = LogisticRegression(fit_intercept=True).fit(results_matrix_train[:,:-2], labels_train)\n",
    "        # reg = ExtraTreesRegressor(n_estimators=500).fit(results_matrix_train[:,:-2], labels_train)\n",
    "        pred = reg.predict(results_matrix_test[:,:-2])>0.5\n",
    "        corrs[i] = np.sum(pred==labels_test)/len(labels_test)\n",
    "    return corrs.mean(), corrs.std()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Combine the best features of all categories"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We use the VE assessor, with $d_{mean}$ and combine it with the different BAM assessors."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_combined_matrix(clmlist):\n",
    "    \"\"\" Combine some columns with predictions from different assessors, to obtain a better prediction.\"\"\"\n",
    "    return numpy.stack([results_dict[v] for v in clmlist]+[results_dict[\"ground_truth\"]]+[results_dict[\"folds\"]]).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "VE+BAM (OC): Acc: 0.87\n",
      "VE+BAM (OC): Standard Error Acc: 0.06\n",
      "VE+BAM (OC): Corr: 0.87\n",
      "VE+BAM (OC): Standard Error Corr: 0.03\n",
      "VE+BAM(JVL): Acc: 0.87\n",
      "VE+BAM(JVL): Standard Error Acc: 0.06\n",
      "VE+BAM(JVL): Corr: 0.83\n",
      "VE+BAM(JVL): Standard Error Corr: 0.03\n",
      "VE+BAM(JVL,Crop): Acc: 0.87\n",
      "VE+BAM(JVL,Crop): Standard Error Acc: 0.06\n",
      "VE+BAM(JVL,Crop): Corr: 0.83\n",
      "VE+BAM(JVL,Crop): Standard Error Corr: 0.03\n"
     ]
    }
   ],
   "source": [
    "oc_hy_matrix = get_combined_matrix([\"ve_dmean_weigh\", \"bam_oc\"])\n",
    "jvl_hy_matrix   = get_combined_matrix([\"ve_dmean_weigh\", \"bam_jvl\"])\n",
    "jvl_crop_hy_matrix  = get_combined_matrix([\"ve_dmean_weigh\", \"bam_jvl_crop\"])\n",
    "\n",
    "for s,matrix in zip(['VE+BAM (OC)', 'VE+BAM(JVL)', 'VE+BAM(JVL,Crop)'], [oc_hy_matrix, jvl_hy_matrix, jvl_crop_hy_matrix]):\n",
    "    with np.printoptions(precision=2, suppress=False):\n",
    "        acc, stderr = cross_combined_accuracy(matrix, n=5)\n",
    "        #print(matrix.shape, )\n",
    "        print(\"%s: Acc: %.2f\"%(s,acc))\n",
    "        print(\"%s: Standard Error Acc: %.2f\"%(s,stderr))\n",
    "\n",
    "        corr, std  = cross_combined_correlation(matrix, n=5)\n",
    "        print(\"%s: Corr: %.2f\"%(s,corr))\n",
    "        print(\"%s: Standard Error Corr: %.2f\"%(s,std))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compute the weights for the hybrid assessor.\n",
    "In this final section, we show how the hybrid assessor can be implemented and used to make even better prediction."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_model_weights(assessor_list):\n",
    "    \"\"\" Return the weights of the model to use in the hybrid assessor.\n",
    "        No Cross-Validation.\n",
    "    \"\"\"\n",
    "    input_matrix = get_combined_matrix(assessor_list)\n",
    "    print(input_matrix.shape)\n",
    "    reg = LinearRegression(fit_intercept=True).fit(input_matrix[:,:-2], input_matrix[:,-2])\n",
    "    return reg.coef_, reg.intercept_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(260, 4)\n",
      "Hybrid assessor weights: [5.66875056 0.12422668]\n"
     ]
    }
   ],
   "source": [
    "hybrid_weights, bias = get_model_weights([\"ve_dmean_weigh\", \"bam_oc\"])\n",
    "print(\"Hybrid assessor weights:\", weights)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use the hybrid assessor class."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {},
   "outputs": [],
   "source": [
    "from coherence_assessor import HybridAssessor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [],
   "source": [
    "hy_assessor = HybridAssessor([VisualEmbeddingsAssessor(device=use_device, coherence_measure = \"mean\", cam_weighted=True, model=model), BestAlignedMeaningAssessor()], hybrid_weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "260it [00:28,  9.12it/s]\n"
     ]
    }
   ],
   "source": [
    "hy_assessor_results = np.zeros(len(eval_concepts))\n",
    "for i, my_concept in tqdm(enumerate(eval_concepts)):\n",
    "    hy_assessor_results[i] = hy_assessor.assess(meaning, my_concept)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7f897902a760>"
      ]
     },
     "execution_count": 210,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAESCAYAAAAR2wXeAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAqcUlEQVR4nO3dT2wc2Z0f8O+PpChaIuUWKUUzgKGhKGP+7CUxRWlnrl7SMZJFDjZljXfh066b2UtOa3FkTBBsYFghfQqMwEtqNjltVhrSCwTIBoHJySUGZiyRnD3ZPow4GmWA3TWHZM+QEkiK7F8OVdVqtfrPe2S9rldV3w8gqKtZrH7V1fzV69/7J6oKIiLKh46kC0BERO3DoE9ElCMM+kREOcKgT0SUIwz6REQ5wqBPRJQjDPpENURkqN7jGI8/LCKbrY7t4rWJGPSJqoSBdqb2cZxUdQXAhmk5iOLEoE+ZJCLXRaQoIjNVzxVFZFxEZkRkNHxuSkRGRWQu3G0UwIiIFGseR79/XUQWRGQo3F4On7vfoAzLVf/P1e5TU66p6LVqX5soLsIRuZQ1YUAfVtVpERlW1ZXwuTFVnYweA7gDYEJVJ8KbwwKAFQALqnoxrG1Hj8cBQFXnRWQKQAHAFID7AE4D6FfV1ZpyDAFYVtXT4fZ9AJPhMe6HZRgKy3C1ap+rAErRa7t7pyiPWNOnLFoCcENEpsJUChAE0gUAUNVFVZ1U1ZUw4Ee1+mbGAAyF+94HMIcgRVNS1VJtwA9t4Nk0zjyAyzX7XAVwr2p7EcC11qdIdDhdSReAKG6qWhKRCwDmRGRZVS8B6EdQO68QkQKC4D0B4FKLw/YDmFPVxZrft7Ve57mBqselBvsQxYI1fcocERkPa99jVU8vALhRtU8RQBHAalhL76/at97jewjSOdW/b6L6WMMIavvV5gCMV20PVe3TD6KYMehTJlU11s4AgKrOAlgNu0rOAXgXQSplNMzRryKo8W+E+03VPJ4FsBH+/kL4+98BUIgahRsoRI3KAGZUdVVEhhEE9/Hwm8NM2Ig7DuCOqkY3otXw+ULc7w/lFxtyiRwJg/UyG2PJJ6zpE7kzgqDxt5B0QYgirOkTOVI1RqCkqpOJFoYoxKBPRJQjTO8QEeUIgz4RUY54PzjrzJkzOjg4mHQxiIhSZXl5+TNVPVv7vPdBf3BwEEtLS0kXg4goVUTkk3rPM71DRJQjDPpERDnCoE9ElCPe5/TrefLkCT799FPs7OwkXRTyRE9PD77yla/g2LFjSReFyGupDPqffvop+vr6MDg4CBFJujiUMFXF+vo6Pv30U1y4cCHp4hB5LZXpnZ2dHQwMDDDgEwBARDAwMMBvfpSIclmxtrWLtMxukMqaPgAGfHoGPw+UhHJZ8d1bH2D5k01ceuk0/ub7r6Ojw+/PYipr+kREPlh/tIflTzaxX1Ysf7KJ9Ud7SRepJQb9Q1hcXISIYHZ29rnt6PHk5NNJFa9evYrp6em6x5qYmGhLmW3Nz89DRLC4WFkdEBMTE5ienq78bHp6GrOzs5icnKy8F/VE78n09DQmJydx9epVlEqlZ/apfR+i36l+/Wi/06dPY3W13pK0RO11prcbl146ja4OwaWXTuNMb3fSRWpNVb3+d+nSJa3161//+rnn2g2Abm5uVrYLhUJle2ZmRoeHhys/m5qaqnuM+/fva6FQ0IWFBZdFPbTR0VGdmZmpbF+/fr3yOPjoPHX//v3K4+r3JVL9/hSLxWfek0bvw/DwsI6Ojj5XptrnIj58Lih/Dg7K+rsvdrRcLiddlGcAWNI6MTU3Nf24G1sKhULDnxWLwfKps7OzmJ6exvXr1+vuNz8/j6mpKczMzFSeW11dxfz8PCYmJlAqlZ7bLpVKmJ2dxcTEBGZnZ432j2rnUW06+v2xsbGmNebJyclnyjYwMFB3v9nZWQwNDVW2b968+VxNvtrq6iqGh4ebvg8AcO3aNSwtLVXKGJWbyCcdHYKzfcdT066Ui6AfNba8cfM9vDn7AcrleAJ/FNSnp6efC3Jzc3OYnJx8JhjWUywWsbi4WPn9xcVFrK6uYmpqCoVC4bntmzdvAgAuXbqEyclJo/2Hh4cxPj6OQqGA6elpDA0NYWNjAwsLC03LNzo6ilKphJWVFczOzmJ8fPyZn0fnv7CwYPR+3bx5E1evXsXGxgZGRkaavg9AcGO9ceMGpqaC9chLpVLTmy0RGahX/ffpXxzpnd99saMXb/ydvjT5P/Xijb/T332xY/X79VSnK+ptq6oODQ1V0h4zMzNaLBYrKZKFhQW9fv16JRVUne4YHR3V8fHxutv1UhvN9h8fH6+kTRYWFnR8fFyXl5e1WCwanefMzIyOj48/k9pRfTa9E53j1NSUjo+P69DQUKUMy8vLz70/0X6t3oeZmRnd3NzUQqGgc3Nzury8rAsLC0zvEBlAntM7LhpbmqUvIhsbG5XHxWIRMzMzlVrrysoKpqamUCwWcevWrUoNfnV1tVIDn5+ff267UChgfn6+sm+r/YeGhrCyslLZf2xszOo8oxr45cuXG+4Tvdb169cxNzeH8fFxzM3NYW5u7pk0TvX+Ucqm0fsAPK3Zj46OYmZmpu6xiMhOavvp2xAR/M33X8f6oz2c6e0+cu4t6lHy7rvvPpOWiLaBIJiVSqVKMKw2OzuL+/fvV7ajFMvk5CQuXrwIALh48SJGR0fx7rvvPrM9PDyMsbExLCws4OrVq5Xg2Wj/8fFxTExMVMp4/fp1TE9PY2lpqRJUFxcXMTU11TBNUywWn0ntRDed2dlZ9Pf34969eyiVSs+lf6rfr6gtolAoYG5uDrdu3Wr6Ply+fBl37txBsVjEjRs3Kuc5NzdXyfO3Sp0RpVW5rLHFq1rer5E7MjKitfPp/+Y3v8Frr72WUImyaX5+vmHQTgt+LigL4hrwJSLLqjpS+3wu0jvUWtoDPlFWuB7w1fagLyIFEZkTkU0RmWr36xMR+cz1gK8kcvqjqnoVAMLAP6OqHF5JRIT42yBrtb2mr6rzVZtLhw34vrdFUHvx80BZ4nLAV2I5fREpAFhp8LOiiCyJyNLa2tpzP+/p6cH6+jr/0AnA0/n0e3p6ki4KkfcS670jItdVtf4sZFXq9d7hyllUiytnET2rUe+dRPrpi8gogNnwcUFVSza/f+zYMa6QRER0CEn03hkHMANgTkSWAYy2uwxERHnV9pp+2JA733JHIiKKHQdnERHlCIM+EVGOMOgTEbVJ3Is5HUYuZtkkIkpaXBOpHRVr+kREbeB6IjVTDPpElFqu0iU2xzXd1/VEaqaY3iGiVCqXFW/OfoDlh0G65HZM6RKbNIzNvq4nUjPFmj4RpdLa1i7uPtjAQVlx9+MNrG3txnJcmzSMbcrG5URqphj0iSiVauNmqzjqIg3jS8rGBtM7ROQV0/Vhz/Ydx5XB05XUytm+402P6SIN40vKxgaDPhF5wzY43y6+YRRw66Vhmt0kojSMCZt9fcD0DhEdioueM65y5GlMw7jCmj4RWXM10CgKztFx4wrOaUzDuMKgT0TPMMmp26ZLTLkMzq7SMKZtEL5g0CeiCtMavKsaedr4MrWCDQZ9IqowrcG7qpHbDrhKupbt6huPS2zIJaIKmwZPFwONbAZcRbXsN26+hzdnP0C53P6ZK9PYQMyaPhFViAj++k9+Hx+tbePlc71trz3bDLjyoZadxgZi1vSJqKJcVvzxX/0Kf/jTX+K7t34VW+3ZtHtnNOCqU4Arg80HXJ3p7cbw+dPo7BAMJ1jL9mFqBRus6RNRhYvas6sBV8H9Q4MHqlBtPRUDsaZPRFVc5KhdDbiKjnugSHR++rRhTZ+IKlzkqIM0TAHLn2xi+HwhtjRM/4lj+FJ3J7Z3D/Cl7k70nzgWy3GzjjV9ohxIcm3W4CUlyL2IIK4ifPZoD9u7BwCA7d0DfMaavhEGfaKMC/q+v4/Xf7yIazPvN22ctdnX1PqjPSw/3MSBYXrH9AZV+x0kDel8HxZGZ9An8oiLoLC2vYu7D4Lc990Hm1jbbtz33WZfU/0njuFEdycA4ESLNIzNTSfo6dOPzg7BlQv93g+K8mFcAcCcPpE3XA3pr72BNLuhuKg9bzx+gkc7+wCARzv72Hj8pGGAjm46wNObzrlTPfXLKoLbxfT0kfdhXAHAmj6RN9Yf7WEpDApLMfZG6agJhrXb1VzUnvtPHMPJnqB+ebKnq2lN3/amk6Y+8r6M3mVNn8gTURpka2e/ZRrERhTIo/lsmgVyF7XnjcdP8HgvaHB9vHfQtKb/dDWsEi69VPA+ZWPDl9G7DPpEoaQn79p4/ASPd4M0yOPd5mkQG7aBPO4piM/0duPS+dOVm06zGu7TzJNWtuO6FElfX8CPVbaY3iGCH41sZ3q7MTLYj64Owchgf6xf/5NMg9QbOdvI+qM9rDws4UCBlYel2FJcPlxfX7CmTwQ/Gtl8+foft3qBvNF7azuQa3+/XJkcrqOjcR3Wh+vrC9b0ieBPI5urGrmLrqCmx7R5b20Gcu3vl/G1Hy3gm//5/+Kf/8cF7O+XYylD1rGmT4Ts1rIBN11BbSdRM52uOaiRbwTz6TzYaFoj/2htG1thV9CtnX18tLaNV188deQyZB1r+kShNHX/s2E74Vncx7SZrrn/xDGcOB7URU8cb9698+VzvegL9+3r6cLL53pjKYMtH0bZ2mDQJ3Is6aDgIrVhc0yb8Qf1unc2Jnjl3EkIgFf+2Uk069Xv4sYHpLOBmOkdIod8WDjbRerK5pg24w/O9HZjxHDB9bXtXSw9/BwAsPTw86ajd10t5J7GBmIGfSKHfAkKLvqHmx7TZvyBzc3EZvSuqzYbVzcTlxj0iRzyJSgkOTApGn9g+h6Y3kxsRhoD4XuwvYuBk8diew/S2AFA2p1nFJFhAFOqOmay/8jIiC4tLTkuFZE7SY8E9SHFZPMe2Oxr2k8/6t65tbOPvp4ufPj2GLq6st2kKSLLqjpS+3zbz1pVVwA8VxCirEq6V5DLRkzTBmrT98CmYbRcVvzRO7/Cv/7pL/Fmix459bp35pVR0BeRH4jIL0TknojcEZF/ccTX3Tji7xORIRe9d1z1WrG5Qa1t7eLugw0clBV3P97A2lbjuf9fPteLvh6z7p1Z1zSnLyJfBvAdAIsAZlX18/C5UREZUdV3XBRKRIoAigBw/vx5Fy9BlBsu8s6uGqhtpmFQaNPtah0dHfjw7TGjVFDWtTrzC6p6S1U/VNXPAUBVP1fVn6vqOyIy6KJQqjqrqiOqOnL27FkXL0HUNkn303fB9tuD6Xvgaj1dAOjq6sCrL57KdcAHDHrvNEjljKjqO6r6IPYSEWWIL42ocZfB5tuDzesHk7MF6+mutPgGYbM4DD3V6pb3DoB/C2AewI3w8Q8BXD3sC4a9d4bC/4kyzVUjaprKsP5oD0sPNoIRueF8Oo3YfIM423ccl18qoAPA5YwtuOJS05p+1N1HRL6tqj+PnheRm4d9wbD3Dm/JlAu2/fRddO90MVbApvYezaeztbPfcj4dm28QBweK3/7jNsoAfvuP2zg4UHR1MbS0Yjo4a0hEvg5gFcAYgFEENX/KiKT7kmeVbRrkzdkPKoONbrdIw5heM9uGXJPj2jTk2iyXCJgPzvpobRtb4Ujfrd3ms2zSU0YtGqr6EwCXAMwCGEbQo4cywpdJo7LY4AmY91G36YIY3CDex+s/XsS1mfdbXrO4+8nbpGGi+XS6OgQjMY5KZjfMwzGq6YvI9wG8paoDInIBwAUAHzstGbWND/PDuGrwTNM3mNriNSvu2vYu7j7YBADcfbDZdLIxG6afBas5chxNVdDR0YHlH45i+eEmrlw4nfteOaZM36VLAN4CAFX9GEdoyCX/+LCqkIvGRl++wZgK5pI5jU4Brgw2n0vGZrIxGzafBZuRxi5GJZfLiu/9t7v43n+9iz96567319cXpjn9ZYTL04vItxHk9SkjfJg0ykVjow/fYGyICP77n75utLqT7WRjNmVIywpTLq9vmr4h2jIN+ncA/FBEfgHgEYKGXMoQF1Pv2nBx4/FlhktT0epOpksQ3i7Gf6O2KUPSXF1fH8ZWuGQa9N8D8H1VfQuodNlk7x2KVdw3Hhe9VlyyrbnavF+m52ZThqTfL1ffStL2DdGWaU7/PQDviMjPROQUgrl4iLznYnZHV1y1rdicm2kZXL5fpr24XK1760Mbl0umNf27qvqWiIwDWEGQ43/PXbGI2suH2p2rthWbczMtg6v3y3bKBhdl8KGNyyXTmv6AiJxS1XlV/SrYXZMyxpfanYteLsHMlafR2SEYtli5qlkZXL1fNr24XF6zpNdAcMmopq+qt2qe+rGDshAlJk29VmwFWRINHqhCtfkYABM+rDmb9Rq5K63m0/+Zqv6ZiPwlgr76QNAl+AKAAdeFI2oXX3qtuGgcDWauLOFAgZWHpdjSIC56fNkG8qR7naVRq/TOW+H/cwBGVfUygD8Ap2GgjLEdHLa/X8Zv/+ELlMvl2MrgqnHUl9SVqSynVnzQapbNaOGU96qfE5F7rgtG1E42aQVXi2z70jCZdFfMtEnb+9UqvfOfEAzEUjw70pvpHcoUm8BYb5HtOGZ3dDmYzDQNwjmQ7KRxIFerhtyFaEBWNRH5nqPyECXGNDAODZxoun1YPjRMuvi2kcbAaMqHrr62mn4njdI6IvItEbkT/QPw79pSOiIPra4/brp9FEnns13k/5NeucultLWXAOaDsy4CiFI98+DcO5Rj0TzuUU4/S/O4cw4kOz58O7NlGvRLADbDx18DMA6gtu8+US50dHTgw7fHKn3645zH3Yfcd9JzIKVN2rqNmn5a3wUg4QpaFxGsoEWUW11dHXj1xVOxB/yk5/9xJem0FT1lulzi5+HiKdHSifedloooh3wYK2Ajq8tbZp3pcol/iSCPvwmOyCVywuVYgbjTRlnukeOSD+k745x+ONEaAEBEvuyoPERGfPjjMWVaVldjBVwE6DR2VUyaLzdK04TkL8Jum98SkW8h6MlDlIg05b5ty2qa+375XC96j3cCAHqPdzbtQeSiy6RtV0Wmgvzpumpa059GsHDKerjN1A4lxodapouVqGyOWy4jXLX66XajNmUXXSZtvpX4UsNNmi9dV43XyA0bcAEA4epZRIlI+o/HJojZlNXmuB+tbWN77wAAsL170DS946rLpGlXRR9u0j7wpeuqadAfE5FRBP31o4bcy64KRdRM0n88Llaisj1ulN7Z3j1omd4Bku1LbnuTTlN7jS0f+vSbBv3/AeB/VW0POygLkbE0BTHTstodV/B7L57C8icl/N6Lp/DsfIh+YSrIL6ZB/9+o6n+JNkRkvdnORFnm6puG7beCYGEUjXVhFFeYCvKHae+dj8PJ1n4W9tnnouiUa65GmJoe14eJvvb2DvD+R5/h4OAgtmP6cF5ZZ1rTXwawiqD3joATrlFK7O+XncyRk7Sk2zX29g7w2n/43zhQoFOA3/zFN9Hd3Xnk42Z5rWJfGC+MLiI/ADCCYI79n7T6HaKkuVrhyhdJtmssP9zEQdhl9ECD7Te+eubIx7VdqzjLjb6uGP0FhCmdAQQTrYmI/LnTUhG1YDLYp96o1WaSnssGSM8gpisXTqMzjLGdEmzHwWYAU5oG6fnENL2zoKo/Dx+/JyJfd1UgolZMe3jYzHvv8luB+YCr9PRc6ezsxG/+4ptYfrgZ3AA6j57aAex6MLHR93BMg/5QVaAfRjC98v9xUySi5kz/2G3mvXe17q1NIHc1eteV7u7OWFI61WzaKpIepJdWpkF/FsANAEMA7qvqn7krElFzNn/s0bz3rbhaDcsmkLsavZs2pm0VSTdmp5VpQ+7nIjIDoAAAIvKnqvqOy4KRv5KuYbr4Y3e1GpZNIHc1eteVpD8HgB8jXNPGdD79XyCYS38TQD+CaRgY9HPIlxqmiz92028FNmxvUDajd4fPF7D8ySaGzxe8nn/oMMdO+maSZTYNudUTrh1pPn0RmQKwAGBYVaePcixqLx9qmDZcBRCb47q4QQWdewSQ4J9q8LBdXH0OfKlUZJnpd9jLVSNyf4ZgmuVDEZFxBO0Ci1XblBJpGjHpqkuf7XFddMMMpmHYxEFZsZLA3OyuPge+zDmfZcY1fQBLVdsrR3jNMQAzVce5CmC+egcRKQIoAsD58+eP8FIUtzQ1nrnqDWNzXNuaq2kZku654upzkPR55UHToC8i31LVv1XVWzU/+lBEBgFAVR9YvuZQ1eONmm2Ex5xF0GMIIyMjHHHRBkmnK1xw1RsmyKefxvLDTQzH2Jfcpgwub76mnwUXn4M0VSrSqmnQV9W/Dadf6AdwL3y6H0E//Tuq+veHeM3Vqsf9NduUgKzmUV31hgmyNBo8UG2aT0/bYCMfPgtpqVSkVcv0jqr+REQuIJhkrQBg5YjdNRcQzOGzgmCg19wRjkUx8CHYuGLXG8a89h5Ma4yW0xq7GmzkKjhn+bNAAdN++h8DqE3xHIqqzovIVLgS10DUoEvJ8SWPmmRXPVe1dxs+9NP35bNA7pg25MZKVSfDhwz4HvAhj1ouK96c/QDLD4Ngc7vNaQVXtXfbGrmbVbbM+fBZILdMB2cNHqLBllLEVR7VtPa+trWLuw82AAB3P97A2tYuzn2558jHNeVqCURXNXKXwZk59WwzrenPAPiX0YaInFLVL9wUibLCrjdK8+3DHtdUGrsgMjjTYZgG/Y9F5A6CLpYC4BKAy85KRQ2laYi6TS33bN9xXBl8GhybBTNXtWd2QaQ84HKJKeJDdzobtpON3S6+4aSXS9IBlzVy8gmXS/SESXBKW3c6V5ONmR43bTdJonbgcokeMJ3LJU3z3kSiQB53LdvkuC7ncUnLsoaHkeVzIy6X6AXTGjzzw3ZsBlzZyPI3iCyfGwVMZ9kcEpGvh//+HMEkaRQTmxq8q5pzFqkC5fIBymVF+eAAcVVcszwTZJbPjQJcLtEDaavB+9A4auKftnaw9PBzAMDSw8/xT1s7ePHLXzrycdM4anV/v2y0Klgaz43smAZ9VdW3gGDmTfbTj19aenik6et/qaaWWnq0F0vQT9tNen+/jK/9aKGy/u+Hb4+hq6t+4E/buZE90/TOlIicCvvqXwbwHYdlIo/Zfv1PslHwlRf60Hs8qNf0Hu/CKy/0xXbsNKXZPlrbxtbOPgBga2cfH61tN90/TedG9kyD/iKACQCnVfUGgr76lEM27Q+uVq4yJ3jthT50CPDai33I68f25XO96OsJbn59PV14+VxvwiWiJJmmd1YA9KvqN8JplimnXM4EGXdbwfqjPXz4/0ooK/Bhi0nUsqyjowMfvj1mlNOn7DO9+utVq2d9DcAdR+WhFDD9+p/0t4Koy2Znh8TaZTONuro68OqLpxjwybimPyUikwjm1P8YwYpXR1lIhXIg6fnhbebIJ8qLw+T030Jek6Mp46oR1cVxXdTK682RT5R3pkF/BcAmc/ruxB1IXTWi2hzXZt+gMl6GlhVaLrccSGXyfgU3kgI6BRg+X2h5I+H0A5QHpkF/FMBU1fZ9B2XJLRcB2tXISpvj2uy7tr2Le5+UUAZw75MS1rZ3G+4brbL1+s33cK3J+/U0dkvNdv1jJtvTiKg9TIP+JQBvAZX1cnM7DYNNbdB0XxcB2tXkbDbHtdm3Nl/YLH8YrbJ1UNbKKluN99vEgSruPthsuB8QXIOl8BoscXI2yjCb+fQVAETk2wDGnJXIYzajUW32dTH0XUTw13/y+5VuenENtLFpnLXZd+BkN/qOd2FrNxg1OnCy2dz7zbdt9wOA/hPHcKK7E1s7+zjR3Yn+E8ca72whTSOYKR9Mg/67AG6IyC8AbCKnQd+mh4nNvi6GvpfLij/+q185CTY2U0aY7rvx+AkePzkAADzeO8DG4ycNf2/gZDdOdnfg0V4ZJ7s7Gt4gbFbj2nj8BI93g1Grj3f3m76+jbStgUDZZ5reUVV9S1W/geAGsO6wTN5yldoA4h/6nrbZEs/0duNS2Oh6qUWj62eP9vBorwwAeLRXxmcNzi1ajeuDH47izsQbLVfjGhnsR1eHYGSwP5F0GFE7HKaf/iqA08hhP31XqQ0X0jZbYpDuliAHI9K0T71N/j/u1bhsJf05IKplGvSfmXtHRL7vsEyZkeTMmWkLNkGf+k0clBUrRouo92P5Yeu0jc3UDj7MdJqWaaspvTj3Dsz/0NLWKOdDEDNlv4h6OtbIddX4T3RYpkE/s3Pv2Pyh+dAol7aaoGl5XSyi7sP1ctX4T3RYh5lP/woyNJ++TYNn0o1yaRtAZFveuBuzba+Xq+klXDX+Ex3GYXL617KU07dNKySZJ09bTTAa8HRQNeAprvKafIOwuV6uUitpavynfLDJ6Z8Oc/qDAM66K1J7uUgruJK2Hjk+DHgyvV6+3FDT1A5D6WQa9IcBfENExhD0kLsA4MfOStVmNn9oSebUXY2ydWXj8RM8Cpfpe7STzIAn0+vl6obKxlnyjWnQHwJwE8HEa/Ph/7mT9B+wy1G2LvSfOIaTPV3Y2tnHyZ6u2Gr6pgHa5nq5Sq348g2CKGIa9EsIpl8Agt474wgGauVK0n/ALnPkLmw8foLHe2ZTK9gwDdC218tFaiVtKTnKPtPeO+8CgKr+BMBFALPOSuSxpHtXRDlyALHmyF0509uNkfD9Gon5/TLp6ePyepn29IluUO/f+APcLr7ufUqOsk98n+51ZGREl5aWki5Gxf5+ObEFpte2dvH6jxdxoECnAB/8cLStvWF8Om6Sr590mo/IhIgsq+pI7fNcJdlClFP/w5/+Et+99atY+8mbrgTlYlIwl/3/bfreu+gnH3fffyB9k9kRVTPN6RPCnPqDDRwosPRgI7acumnN0VXvnaTbKoB01Z6Zp6c0Y9C30H/iGE4cD3qjnDgeX28U0wZaV713XAYx0/SKqxuPi/QOB1FRmiUS9EVkGMCUqnqxGItpYHDVG8V0EJNtYHQ1742ppFcPC9bSfb9yzNvFNxJZSIbIJ4kEfVVdEZHnGhiSYBuYRhzUiE1XbbIJjLbpEhdBLOnVw9a2gzVyAQRr5G7v4typnob7J93oTNQOSaZ3NhJ87YqkAxPwtIG2VTC3eX0f8vS2tfe4bzw2i62kqU2B6Ci8zOmLSBFAEQDOnz/v9LVcBiYX6RXT1/ehsTHp3LfNYis+3CSJ2sFJP30RGQfQX/P0qqouVu1zX1UvtjpWO/rp23ytT9OCK1nte2/DdFyFquLN2Q+q8v8cSNVKmj4HedSon76Tmr6qzrs4riumtee0LbjiIk//tHG0hEsvFVo2jiYZGGx6OyX9rSRtfKjU0OEkMjgr7L0zFP6fGi4XXHExMMmF332xg7sPNnGgirsPNvG7L3Ya7ms76Cvu98B2EJWLgVxZxQFq6ZVY7x00b1fz0pnebgyfL2D5k00Mny/EtuCK7TqqSdZGNx/vPbf9QuFLdfe1nQL5zdkPKvn32zHUHH1o1/BF3J8bvrfp5WVDbhxcBMegAiqABP9Ug4eN2CzgYTo4K+mv1K+80Ife453Y3j1A7/FOvPJCX8N9bQLD2tYu7j4IOnTd/XgDa1u7OPflxt0rTTBlE3DxueF7m16ZDPquguP6oz2sPAyC80qMg6NcDc5yoaOjA3//779h1Dhqt1Rg8+3Dl5eDqFx9bvjeplMmJ1xzlW+0ydPb5LPrDc466uu71NXVgVdfPGU0y6hpnjzoXnkanQJcGWzevRIIeuX89h++QLlctip7HvnyuSE/ZLKmb5N7t+FqcJSLwVkuuZrP5nbxDaPj7u+X8bUfLWBrZx99PV348O0xdHVlsv4SC18+N+SHTAZ929y7DZvBUS4afV19pbYZf5D0fDYfrW1jK1x7d2tnHx+tbePVF0/FUoasYiqGIpmsHq0/2sNymHtPqjtZvRtPu5mmQGxSUdF8Ngf6dD6bdnv5XC/6eoL6Sl9PF14+19v2MhClVSZr+v0njuHEsU5s7TZvGHXJptHXRZdNmxSI1fxDLbbboaOjAx++PZbYCmZEaZbJv5bPtvewtfv06/9n2+2v6ds0npk2PEd92V+/+R6utaiR10uBxFHWaD6bzg7BlQv9LVMGrgad2TQmU/LSMvgwDzJZ03fV/Q9wM4maaX92m77sXz17Eh0ClBXokGA7jrIGDa7xDzqj7OLnwC+ZDPpR97/oQxbn4uEu5qg3Dbq1taRmtabPtvcQfREoa7DdaOSsTVlt9vVhXAElj58Dv2Qy6Nt0/7Ph8sNrEkhrbzDNbjg20yW4YjtUP+kpJmykqaxJ45QNfslk0AfcdFFL+sNr8w3GZroEV1zNP5S0NJXVBxwn4JfMBn0XbD+8cdcGbb7B2EyX4KKsT8uRbCrIxXkxXWGP4wT8wa4PlkynFbCdVjju1wfMe7i4KqsNF1MFuDovTmtAacaaviNpqg36UFYXKQBX58V0BaUZa/qw60Nsum+aaoO+lDXuRUxcnhcXXKG0crJGbpxcr5FrOxrWpgEvTT080lRWG1k9L6JWGq2Rm/uavs00zFlefi9NZbWR1fMiOqzcB32bFIAvaRAiosPKfXoHsEsBMF1ARGnA9E5MXKULOCEVEbVD7rts+jC60sXUykRE9eS+pu9qPV0XZfBhEBURpVvug74PjbOmZfDhBkVE6Zb79I4PoytNy5D0hG9ElH65D/qAH5NBmZTBhxsUEaVb7tM7aWPTe4g9goioFmv6yGaPGB96JRGRf3If9KPFxpcfBsHxdkaCow8zZxKRf3Kf3okWGz8oa2Wx8SzwoVcSEfkn9zX92mxORrI7bPQlorpyX9OP1p3tFODKYPN1Z9OGM0wSUS3W9C3WnSUiSrvcB33Aj376RETtkPv0DhFRnjDoExHlCIN+ynCULREdBXP6KcJRtkR0VG2v6YtIQUTmRGRTRKba/fppxqmVieiokkjvjKrqVVU9DaAoIkMJlCGVOMqWiI6q7ekdVZ2v2lxS1dV2lyGtOMqWiI4qsYZcESkAWGnws6KILInI0traWnsLlhA20BJROzip6YvIOID+mqdXVXWxaruoqpP1fl9VZwHMAsDIyEjmo6BpAy0bconoqJwE/ZoUznNEZBRhUBeRgqqWXJQjLUynQeZ0yUR0VEn03hkHMANgTkSWAYy2uwy+MW2gZUMuER2V+J5DHhkZ0aWlpaSL4Zzp6l1ZXOWLiOInIsuqOlL7PAdnecJ00jdODkdER8FpGIiIcoRBn4goRxj0iYhyhEGfiChHGPSJiHKEQZ+IKEe876cvImsAPkm6HEdwBsBnSRfCAZ5X+mT13Hhe9b2kqmdrn/Q+6KediCzVGyCRdjyv9MnqufG87DC9Q0SUIwz6REQ5wqDv3mzSBXCE55U+WT03npcF5vSJiHKENX0iohxh0CciyhEG/ZiIyJSIjIrI9QY/vy4iy+G/QpuLd2giMiwiC01+3vS8fWVwXmm9XgURmRORTRGZarBPWq+Zybml9bpdD8+tUfyI7Zox6McgXA3sfrQGcLhda0BVL4X/Sm0t4BGo6gqAun2FDc/bS83OK5TK6wVgVFWvquppAEURGar+YZqvGVqcWyh1101EhlV1WlWvArhW5+exXjMG/XiMAYiW91oJtyvCD+ewiGjK/sgiGw2eb3reKVD3vNJ8vWrWp15S1dWaXVJ7zVqdW1qvW1gBgYgMI1hKtlas14xBPx7VNY6Nmm2o6qqqjgG4BOBWmr52ttD0vNMqC9crLPNKnR+l/po1Orc0X7ewrDcATNb5cazXjEE/HtU1jv6a7Yrwjv4uUviH1oDReadVyq9XUVXrBZAsXLNG5wYgnddNVUthemclrPFXi/WaMejHYwFP88PDAOaa7Rx9ncsAq/NOq7RdLxEZRTiwp05tN9XXrMW5PSNt1y20iueDeqzXjEE/BmGu8WL4gRyoanBZCP8vishM+PN6OTtvhbWOoeraR3Rejc47DZqdV8qv1ziCMs+JyDKA0fD5LFyzpueW1usmIuNhuccBLEQN0K6uGUfkEhHlCGv6REQ5wqBPRJQjDPpERDnCoE9ElCMM+pRJaRqYQ9RODPqUOeGkVDeSLkcj4cRhqelSSNnCoE9Z5N2gnHBWz+tAZfTlRNJlonzqSroARI4MhUF2TFXHwscDAG4CWEYwadUwgAkEIx6vIZj3ZAxAQVUnwhTRdwAUEIyS7A9/fi86bvULikgRwbwvQwB+AOBK1e8OAbgcTgo2BOBq+BrFesesmkL3MoCZNA2iIr+xpk9ZtaGq0wBKYaBdAYJaNp5+E4iemwawiCDYTyIc6YkgyC+G/64hnOmw5rjVVgH0h4H7i5rfXQGwGs4MuYSnw+qfO2Z43IHwuWEGfIoTa/qUVaXw/w0Ete1W+63XPqeqq3UWrdio+r/2uBvRzxv8LsKflUSk2TFXEUwRXAAwD6IYsaZPeVRA8xsBgEqKZRX1A2//YX+3Sc+ifqDybWQGwFCz2SSJDoNBn7JoGE9ryiMARsIUSbTcXAlByiWadK2AIHc+Fj6OJmJbRZDzH0WQh/9XAEaiffD8ylujVT+v/d3PwtcfDX9vqLp81ccM0ztTAKbCibgK8b01lHeccI3IM+FsiysI0j39CPL6TPNQLJjTJ/JPP4Ka/iqA+wgWBCGKBWv6REQ5wpw+EVGOMOgTEeUIgz4RUY4w6BMR5QiDPhFRjvx/ekYhwsNbt1MAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "median = np.median(results_dict[\"ground_truth\"])\n",
    "ground_truth_classification = results_dict[\"ground_truth\"] > median # Classification: larger or smaller than median\n",
    "target1 = \"ve_dmean_weigh\"\n",
    "target2 = \"bam_oc\"\n",
    "hy_assessment = (hy_assessor_results-hy_assessor_results.mean())/hy_assessor_results.std()\n",
    "\n",
    "f, ax1 = plt.subplots()\n",
    "ax1.scatter(results_dict[\"ground_truth\"], hy_assessment, 5, label=r\"HY-Assessor, VE+BAM\")\n",
    "\n",
    "ax1.set_title(\"scatter plot\")\n",
    "ax1.set_xlabel(\"human rating\")\n",
    "ax1.set_ylabel(\"assessor (normalized)\")\n",
    "ax1.legend()\n",
    "ax1.show"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Supplement: Compute user agreement, Further Analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We compute the value of Cohan's Kappa. (https://de.wikipedia.org/wiki/Cohens_Kappa)\n",
    "\n",
    "First set up the $d_ij$ matrix.\n",
    "First we compute kappa of the ratings (4 items)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = 10 # number of raters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(260, 4)\n",
      "First lines of dmatrix: \n",
      " [[ 0  0  3  7]\n",
      " [ 0  0 10  0]\n",
      " [ 0 10  0  0]\n",
      " [ 2  5  3  0]\n",
      " [ 3  6  1  0]]\n"
     ]
    }
   ],
   "source": [
    "dmatrix = data_frame.groupby(['source_file', 'source_id', 'rating_num']).size().unstack(fill_value=0).values\n",
    "# The dmatrix contains the number of raters, that gave a certain rating for each concept.\n",
    "print(dmatrix.shape)\n",
    "print(\"First lines of dmatrix: \\n\",dmatrix[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {},
   "outputs": [],
   "source": [
    "pc = np.sum(np.power(dmatrix.sum(axis=0)/dmatrix.sum(),2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Chance of agreement: 0.47008547008547014\n",
      "Chance of agreement on random assignment: 0.2899440828402367\n",
      "Cohens kappa: 0.2537002831633349\n"
     ]
    }
   ],
   "source": [
    "p0 = ((1.0/(d*(d-1.0)))*dmatrix*(dmatrix-1)).sum(axis=1)\n",
    "p0  = p0.mean()\n",
    "print(\"Chance of agreement:\", p0)\n",
    "print(\"Chance of agreement on random assignment:\", pc)\n",
    "print(\"Cohens kappa:\", (p0-pc)/(1-pc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Varianance Hypotheses Test\n",
    "\n",
    "What is the probability that the Variance of the data is observed under the null hypotheses of a uniform distribution?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.25\n",
      "Variance of discrete uniform: 1.118033988749895\n"
     ]
    }
   ],
   "source": [
    "# Variance for a distrete uniform\n",
    "var = (4*4-1)/12.0\n",
    "print(var)\n",
    "std0 = np.sqrt(var)\n",
    "print(\"Variance of discrete uniform:\", std0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variance of data: 0.9314136025527721\n"
     ]
    }
   ],
   "source": [
    "dmatrix = data_frame[\"rating_num\"].values\n",
    "overall_std = np.std(dmatrix.flatten(), ddof=1)\n",
    "print(\"Variance of data:\", overall_std)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {},
   "outputs": [],
   "source": [
    "dmatrix = data_frame[[\"source_id\", \"source_file\", \"rating_num\"]].groupby([\"source_id\", \"source_file\"]).agg(\"var\").values\n",
    "dstd = (10/9)*dmatrix\n",
    "T = (10-1)*(dstd/overall_std)**2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 246,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.0879007358707273\n"
     ]
    }
   ],
   "source": [
    "from scipy.stats import chi2\n",
    "alpha = 0.01\n",
    "X_value = chi2.ppf(alpha, 9, loc=0, scale=1)\n",
    "print(X_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5846153846153846\n"
     ]
    }
   ],
   "source": [
    "print(np.sum(T<X_value)/260)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Speak: for 58% of the concepts, the variance of the rating is significantly smaller (p=0.01) than if the values were drawn from a uniform distribution."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compute kappa for the agreement on word level. One kappa value is computed for each concept."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(260, 18)\n",
      "[[6 0 0 0 4 0 0 0 0 0 0 0 0 0 0 0 0 0]\n",
      " [9 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0]\n",
      " [6 0 1 0 2 0 0 0 0 0 0 1 0 0 0 0 0 0]\n",
      " [2 0 0 0 2 0 1 1 0 0 1 3 0 0 0 0 0 0]\n",
      " [2 0 5 0 0 0 1 0 1 0 1 0 0 0 0 0 0 0]]\n"
     ]
    }
   ],
   "source": [
    "dmatrix = data_frame.groupby(['source_file', 'source_id', 'description_idx']).size().unstack(fill_value=0).values\n",
    "print(dmatrix.shape)\n",
    "print(dmatrix[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {},
   "outputs": [],
   "source": [
    "wchoices = data_frame[['source_file', 'source_id','num_word_choices']].drop_duplicates()\n",
    "num_choices = wchoices['num_word_choices'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Chance of agreement: 0.3752991452991453\n",
      "Chance of agreement on random assignment: 0.10127242572661124\n",
      "Cohens kappa: 0.30490520978404567\n"
     ]
    }
   ],
   "source": [
    "d = 10 # Number of raters\n",
    "p0 = ((1.0/(d*(d-1.0)))*dmatrix*(dmatrix-1)).sum(axis=1)\n",
    "pc = 1.0/num_choices\n",
    "print(\"Chance of agreement:\", p0.mean())\n",
    "print(\"Chance of agreement on random assignment:\", pc.mean())\n",
    "print(\"Cohens kappa:\", (p0.mean()-pc.mean())/(1-pc.mean()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How do the kappas change for concepts with different average ratings?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {},
   "outputs": [],
   "source": [
    "local_kappas = (p0-pc)/(1-pc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "          source_file  source_id  rating_num  kappa_agreement\n",
      "0            awa_Shap          0         2.7         0.377778\n",
      "1            awa_Shap          1         2.0         0.783333\n",
      "2            awa_Shap          2         1.0         0.301852\n",
      "3            awa_Shap          3         1.1         0.012346\n",
      "4            awa_Shap          4         0.8         0.136508\n",
      "..                ...        ...         ...              ...\n",
      "255  places365_mixin5         15         1.5         0.120000\n",
      "256  places365_mixin5         16         1.3         1.000000\n",
      "257  places365_mixin5         17         1.4         0.593651\n",
      "258  places365_mixin5         18         1.1         0.275000\n",
      "259  places365_mixin5         19         0.9         0.135802\n",
      "\n",
      "[260 rows x 4 columns]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source_id</th>\n",
       "      <th>rating_num</th>\n",
       "      <th>kappa_agreement</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>16.157895</td>\n",
       "      <td>0.326316</td>\n",
       "      <td>-0.040414</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>17.650000</td>\n",
       "      <td>0.700000</td>\n",
       "      <td>0.106772</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>14.175439</td>\n",
       "      <td>1.236842</td>\n",
       "      <td>0.269242</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>16.529412</td>\n",
       "      <td>1.698039</td>\n",
       "      <td>0.386514</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>21.034483</td>\n",
       "      <td>2.212069</td>\n",
       "      <td>0.396390</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>25.314286</td>\n",
       "      <td>2.617143</td>\n",
       "      <td>0.504990</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   source_id  rating_num  kappa_agreement\n",
       "1  16.157895    0.326316        -0.040414\n",
       "2  17.650000    0.700000         0.106772\n",
       "3  14.175439    1.236842         0.269242\n",
       "4  16.529412    1.698039         0.386514\n",
       "5  21.034483    2.212069         0.396390\n",
       "6  25.314286    2.617143         0.504990"
      ]
     },
     "execution_count": 258,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "by_concept = data_frame.groupby(['source_file', 'source_id']).mean()['rating_num'].reset_index()\n",
    "by_concept[\"kappa_agreement\"] = local_kappas\n",
    "print(by_concept)\n",
    "bins =  np.arange(0, 3, 0.5)\n",
    "ind = np.digitize(by_concept['rating_num'], bins)\n",
    "by_concept.groupby(ind).mean()\n",
    "#res_mat = by_concept[['rating_num','kappa_agreement']].values\n",
    "#plt.scatter(res_mat[:,0], res_mat[:,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'kappa')"
      ]
     },
     "execution_count": 259,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEECAYAAADEVORYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAlsklEQVR4nO2dX2wdx5XmvxLNAPQCG0qKHsaEGFsCVgEEJiDNRAr0FI8IAWtY4DjyGkL0uGb2PUtLhgVYBhRIMudlnxaWX501YtIJEccBBNLOk7DWgH8gEAKiBSTHEugMoAlNDWbN7BDU2Yfbl7r3svuqDtnndlX39wME3a5b7Fvd1f316VOnTjkRASGEkGqwq+gGEEII6RwUfUIIqRAUfUIIqRAUfUIIqRAUfUIIqRAUfUIIqRBPFd2AJ/Gd73xHnn322aKbQQghUTE/P/8vIrKvtTx40X/22WcxNzdXdDMIISQqnHNfppXTvUMIIRWCok8IIRWCok8IIRWCok8IIRUi+IFcQlqZXlzGxLXb+Gp1Dc/09mD8xCGMDvZtu55lXYvjAoDz00v44MZ9bIigyzmcPrIfF0cHdtwGX3723v/G9Tsrm9vHDu7Br1778Y73qzkuTd0QrgWr60aLyzvLpnNuCMAVERnJ+P4KgBkAQyLyzpP2Nzw8LIzeIXWmF5fxxm+WsLa+sVnW092FSy8PNN1AvvUs61ocF1ATu/c/v7dlH2eO9ndE+FsFv85OhV9zXJq6IVwLVtdNO5xz8yIy3Fqeu3tHRBYAbPmhpBGnANwRkdmGbUK8mbh2u+nGAYC19Q1MXLu9rXqWdTVo9vvBjfup+8gqz5s0wW9X7ovmuDR1Q7gWrK6b7WDl08/q/REAdbN9IdnegnNuzDk355ybe/DggUX7SKR8tbrmVe5bz7KuBs1+NzLezrPKY0FzXJq6IVwLVtfNduj0QO6Bhs8rLdubiMhVERkWkeF9+7ZMKCMV5pneHq9y33qWdTVo9tvlXGrdrPJY0ByXpm4I14LVdbMdOi36dxs+72nZJuSJjJ84hJ7urqaynu4ujJ84tK16lnU1aPZ7+sj+1H1klefNsYN7VOW+aI5LUzeEa8HqutkOnY7emUHN378AYAjAZId/n0ROfdDrSVEQvvUs61ocF4DNgcqiond+9dqPTaJ3NMelqRvCtWB13WwHq+ideQDPJ4O6cM7N1KN5GqJ3RkTk7JP2x+gdQgjRkxW9k7ulnwi9aykbafhcF/rZvH+bEEJIezgjlxBCKgRFnxBCKgRFnxBCKgRFnxBCKgRFnxBCKgRFnxBCKgRFnxBCKgRFnxBCKgRFnxBCKgRFnxBCKgRFnxBCKgRFnxBCKgRFnxBCKgRFnxBCKgRFnxBCKgRFnxBCKgRFnxBCKgRFnxBCKgRFnxBCKgRFnxBCKgRFnxBCKgRFnxBCKgRFnxBCKgRFnxBCKgRFnxBCKgRFnxBCKgRFnxBCKoSJ6DvnrjjnjjvnXs/4fqzd94QQQmzIXfSdc6cA3BGR2Ybt1u97k+/3Oud6824DIYSQdJ4y2OcIgHeTzwsAXgEw1fD9LIB559wCgL+KyKpBG0iJmV5cxsS12/hqdQ3P9PZg/MQhjA72Fd2sjqI5BzxfdsR4bi1E/0DD55WWbYjIqnPuCoBJAK+l7cA5NwZgDAD6+/sNmki2S9EX+fTiMt74zRLW1jcAAMura3jjN0sAEPzNlhfTi8sYn7qJ9Q0BUDsH41M3AWw9B1bny+o6sHqYnZ9ewgc37mNDBF3O4fSR/bg4OrDjtsZ4LVr49O82fN7Tsg3n3AEAvQCeA3Al2W5CRK6KyLCIDO/bt8+giWQ71C/y5dU1CB5f5NOLy7ns+9jlz/DcuU9w7PJnmfucuHZ78yars7a+gYlrt3fchlh4++Nbm4JfZ31D8PbHt7bUtThfVteBZr+auuenl/D+5/ewIbVztiGC9z+/h/PTSztqb6zXooXozwAYTj4PoWbRN3IcwGri1rmS1CERYHWRa27gr1bXUveRVu77IImNr79Z9y7XnC9frK4DzX41dT+4cT/197LKfVnOOIdZ5aGQu+iLyBSAg8654wD2NgzoziRVPmz4vjepTyLAQkAA3Q38TG9P6j5ayy3fSmLC93xpsBI7zfWlqVu38H3LfelyTlUeCiYhmyJyVkRmReRsQ9lI8v9qw/fvWPw+scFCQADdDTx+4hB6uruaynq6uzB+4lBTWayv3j709nR7l/ueLw1WYqe5vjR1rdpr9TCxhpOziDcWAgLobuDRwT5cenkAfb09cAD6entw6eWBLQNnVm8lIXDh5GF072oWrO5dDhdOHt5S1/d8abASO831pal79MDu1N/LKvelL+O6zSoPBYvoHVJS6kKRd9TG+IlDTVEQQPuHyehg3xN/s/fp7lQfd+/T6VZyTGj7wed8aejr7Ul15exU7DTHpan757+mP+izyn3RXrehQNEnKvIWkPo+gXwfJllGZ+Bv3t5Y9IMvlmKnOS7fulZvfVZGkDUUfRIEeYvYw7X0CJescuJPbGL3TMabyU7HooBiH77bhaJPSonljU7iErtY3TBWcCCXlBKrQWcSHxaD2TFDS5+UkthcEMSWmN5MrKHok9LCG52QrdC9QwghFYKiTwghFYKiTwghFYKiTwghFYKiTwghFYKiTwghFYKiTwghFYKiTwghFYKiTwghFYKiTwghFYKiTwghFYKiTwghFYKiTwghFYKiTwghFYKiTwghFYKiTwghFYKLqBBScqYXl0u5gtj56SV8cOM+NkTQ5RxOH9mPi6MDRTcreCj6hJSY6cXlpkXBl1fX8MZvlgAgauE/P72E9z+/t7m9IbK5TeFvD907hJSYiWu3NwW/ztr6Biau3S6oRfnwwY37qnLyGBNL3zl3BcAMgCEReSejzhiAFQCzIrJq0Q5Cqs5Xq2uq8qLxdUVtiKT+fVY5eUzuou+cOwXgjojMOueGnHOnRGSqpc67AN4VkYW8f5/UKKsfV0OZz4HvsT3T24PlFIF/prcndb8WfnLftmpcUV3OpQp8l3M7aqsloVyPFu6dEQBzyeeFZHsT59xxAAcADCefSc7Ub57l1TUIHt8804vLRTctk+nFZRy7/BmeO/cJjl3+bMdtnV5cxi8mbzadg19M3izFOZheXMb4VPOxjU+lH9v4iUPo6e5qKuvp7sL4iUNb6tb95HUxrfvJz08v7aitvteixhV1+sj+1N/LKi+akO5JC9E/0PB5pWUbqD0EZgB8COBs8mbQhHNuzDk355ybe/DggUETy01sflyLG+LN3y5h41GzJbjxSPDmb7cKWL0Nvg+dvB9Q9X36noO3P76F9Y3mY1vfELz98a0tdUcH+3Dp5QH09fbAAejr7cGllwdSLcz/dePelrK0cish17iiLo4O4MzR/k3Lvss5nDnaH+wgbkj3pIVP/27D5z0t2wDQC2BGRFYTN88IgCb3j4hcBXAVAIaHh+mkUxKbH7fdDbHd19//++8b3uUat4JVNIzmHHz9zXrqPrLKRwf7vNr2KONOay3XtDXNtZRVrnVFXRwdCFbkWwnpnrSw9GcADCefhwBMtnw/n5TXuWPQhkqTdZNklRdN0TeExgqzstg04lg0mv7K8rGnlY+fOITuXc3l3btcqisqNkK6J3MX/WTQ9mDir98rIrMA4JybSb6/CmBv4tb5YVZ0D9k+Gj9uCFjcEFnDeWnlGhEr+gEFAL093apyX3q60+WgtVzTX+oom9YOCndcVkVI96RJnL6InBWRWRE521A20vL9VOP3JD80ftwQsLghfna037tcI2IhWGwXTh5OtYgvnDy8o/1eevn7WwRhV1LeiKa/+jLOS1r5xLXbqWMVoY5FaQjpnuSM3JLi68cNgXo78wxnq/t6fcIPx08cavLTA9ki9pPv7WuaCdpYvhM0IYgW50uzX83va85tCG9RloRyTzoJfDLD8PCwzM3NPbkiITvAN4b62OXPUv3sfb09uH7uhW3/fmtagTohR6T4UvS5rSrOuXkRGW4tp6VPCPytMCtrVPNmEhu+51bzVkC2D0WfEAXasEINMYUgWmDltiLNUPQJUUBr1JZQ/N5lhqJPiAJaoyR2KPqEKKE1SmKG+fQJIaRCUPQJIaRCUPQJIaRCUPQJIaRCUPQJIaRCbEv0nXPP5twOQgghHcArZNM591MArwIQ1JKdPgfgh4btIoQQYoBvnP4BAJcAHEdtlSuubUsIKRTNQuOhLEoeAr7undXkHwAMAtiyri0hhHQKzTq9IS1KHgK+ov8hABGRCQAHkaxfSwghRRDCEpex4uveEQCvOOcOAJgXkY8M20QIIW2JbYnLkPC19D9DTfinADjn3H+3axIh5WF6cRnHLn+G5859gmOXP6usSyFvYlviMiR8Lf1ZEfnH5POnzrmXAcA59x9F5F9tmkZI57AY6Kv7kuuuhbovGUBHBxHPTy+VbnEWzbKVTIfdjHf0jnPufzZsDzvnRlCL6jmRf7MI2Tm+Qm4lzu18yZ0S/dZlGDdENrc7JfwWUTa/v/mX1L///c2/bDkupsNuxlf0VwBMohajD9TcPAAwlHuLCMkBjZBbiXMIvuQPbtzPLO+E6Gv6YXpxGeNTN7G+IZt1x6duptZdXVtP/b2scqbDfoyvT/8sgOdRC9V8TkQ+Tf5N2DWNkO2jidiwEucQfMkbIqryvNH0w9sf39oU/DrrG4K3P75l2saq4Sv6s+BALomItHVss8qtxHn8xCH0dHc1lXXal9zlnKo8bzQP1K+/SbfS08p3P92dWjernDzGV/Q/FZF/TKz79wDcBWoDuXZNI1WiyCgXK3EeHezDT5/v2xTYLufw0+ez3Qyac+Bb9/SR/aryvPvB6oH61kuH0bWr+cHVtcvhrZcO72i/VYADuaRwio5y0Q70+UbDTC8u46P55U1XyoYIPppfxvB396T6sy183xdHB/DFg3/D9Tsrm2XHDu7JbG/e/aCJsnm6exe+WX+UWp7GLgAbLdvkyfiep/pA7lTy738k/88atYuUAF+rMaYZk/VomEYhf//zezg/vbSlrtWsUY3ve3pxGQv3HjaVLdx7mNoXFv3wxz898C7/1lNdKTXTyyeu3cb6o5Zz8EiCvGZCw9fSnwHw39CQZVNEfgjg07TKzrkryd8Micg7GXUOALgiIq+oW10yypgMSmM1WgykOgekjVWmubI1lrMmGsZq1qjG962JTLLoB80+H2ZE3qSVhxAZFSu+lv5zqGXZnEMtkicz945z7hSAOyIy27CdBpO2obzJoDRWo4Xf92dH+r3LNZazJhomhFmjGnG0aIPVOQghMipWfEX/IYCvk89PyrI5gtrDAQAWku0mkgfBVGt5FYnJtaFBIzYWA6kXRwdw5mh/0yDqmaP9qb5sjeWsQXNcmrq9PekRKmnlGnG06AercxBCZFSs+Lp3PgSwW0QmnHPjaJ9l80DD55WWbTjnhlB7GKwgA+fcGIAxAOjvT7fYykJZX1Of6e3xDo+0mjF5cXSg0HQDo4N9mPtypWnQNyt6R3MOLpw8jPHJm00+7e5dDhdObo1c0aQgsOgHzT6t6pJmfEV/t4j8GQAS4f+vberebfi8p2UbAK4kZXsAHHfOjYlI00Mk2b4KAMPDw52ZRVIQGnGMCW2+kyJnTPb2dKfO5EyznPsy+qsvpb800TuA/znQiqPvg0fTBg2afVrV1VDGMbZGfN07m4OxiaV/pU3dGQDDyech1KJ+NhGRERH5OYDXUEvkVunc/GV9TR0d7MOllwfQ19sDh5ooXnp5IMib58LJw+huifluZzn79pel6250sA/Xz72ALy6/iOvnXmgb+5/24Il9zMiKso6xNeJr6b+WWPevoOaaeS6roohMOeeuOOeOA9jbMKA7IyJb/PtVp8yvqbHkO7FyK4Tgugsh6VtMVOF8ZYq+c+45AJcbiwDsRW3lrE/RZmF0ETmbfJxtKBtpqbOK2kOk8sQijmXGwq0QguvO6sFTVhdICA9qazJFX0S+cM7NIn0CFhdGJ+QJhJDH3eLBU/QMaktCeFBb09anLyLvicgXKf/e61QDCYmVEMY1LMaMyhpmDJR3jK0RX58+IWQbFO26sxgzKrMLpMxjbHUo+oSUnLwfPGV3gRT9oLaGiekIiZAypqImnYGWPomOskaO+FL0QGoVXCBlhqJPoqJowdNi8YAKIZa87C6QMkPRJ1ERguD5YvWAim0gtepvZqFBnz6JipgEzyq0Maa0wlVIaxAbFH2iosgBRCAuwdM+oHzPbUwDqWWO6Y8Vij7xJgSrLSbB0zygpheXMT55s+ncjk/eTD23IUz68iWmN7OqQJ9+RBTtG7X0p/semzZVcJFo0jBc+N2t1DVfL/zu1o7SMBeNNqa/6Gu8ClD0I8EyasX3RrNM3uV7bNoc9UWiCW1My+ffrlyDhZD67vMn39uH9z+/l1qets+YIrNihaIfCVZWtuZGs5qJqTm2mKJ3gOItcgsh1ezz9zf/krqP39/8y5ZVzWLrWy2hvMXQpx8JVla2ZqBN60/3HZjUHFtZfcS7n05f9zar3PfcWgykavapeYMpa98CYYyH1aHoR4I2asVCcDUDiJqLXHNsMUXvaHjrpcPo7mpZvavL4a2Xtq7epTm3FkKa9rbXrtyXsvYtEFYUE0U/EjRWtpXgarB6g4gpekfD6GAfJk79oOmBOnHqB6kPVM25tejfLue8yzVvMGXtWyCstxj69CNBMyio8Y1qIkw0vlztG4TvsZU574uv/19zbrULufj4neuD6K2klb/10mGMT93E+sbj77LeYMrctyFlJqXoB4AmXDFvUbB6mGgvcovlCjWEMsjmg+bcavrX96Hel/H7fTv8/Xp93/MeU5+FsIpaHYp+wVhEV1gJruZhognVK5rYQgW1AuLbv74Pdavf1xBbn4X0FkPRLxiN9exr2VhZFZqHyR//9CB1H1nlRRJbqKBWQPKeh2EpYL5tja3PgOLDd+tQ9AvG90bTWDZWN6XmYRLSwNWTsIpGscRXQKzmYRRtvYfSZzG5mOoweqdgfKMrQgj5Gh3sw0+f79uM0miXAiGm8DtNNEpsWM7DyJsQrnENIcXea6DoF4zvjaaxnK0uxqwUCGn7LVpANGiiUYDiM41qsJqHYUFMb4dAfA+pOnTvFIyvK0bz6q31d1r4UUMauHoSmmiU2AYQLaOo8kbT1i7nUh/KnXw7i+0hVYeiHwA+N5qVP90q9r7+9xbhd3n7UTXnNrYBxJBCBZ/E+IlDGJ+82ZRttHuXS23r6SP7U6PDTh/Zb9rGRkKKvddA904kaF69Nf70omd3Ajp3lIXrSnNuY7PuinbZqGk11DMM94ujAzhztL9pfOnM0f4tSdwsicmF2YiJpe+cuwJgBsCQiLzT8l0vgPcAHAdwVUTOWrShjPhazpoYecvZnb5YZdnUvBH4nttYrTtfioxGmbh2u2nmLgCsb0iwb1FWLkzrPshd9J1zpwDcEZFZ59yQc+6UiEw1VDkuIq8kdb92zr0rInfzbkeV0cTIW83uBPwvXk34nUWIq4aY3CWAfq2CIscrNAbI+emlJsNmQ2Rzu5PWft5jIJ3oAwv3zgiAueTzQrK9ScsDYI6Cnz9a613zijo62Ifr517AF5dfxPVzL7QVfF83jCZksugQ19jcJZrzUHQ0isZ9+MGN+6l1s8pjoRN9YOHeOdDweaVle5PEzbOQ8d0YgDEA6O/vz7l55cfSevdF44bRhEz6WtqWk3dCmVnpQ0xrFWjeorRhtlbk7YrpRB9YiH6j5b6nZbuRsSx/vohcBXAVAIaHhzvbiwVQZDQKYCNimotXm8DLZ41cy5C+mGZhagwAq/EKTUJBwM8AcQ5I0/dOzqcLIW/WdrBw78wAGE4+DwGYbK3gnDuORNQTi7+yFB2NYoXmVV27VoDPBDErSzC2WZhFr1UwvbiM8ambTedrfOpm5vnydR/2PJUuXVnlFli4YjoREZT7GUp89gcTYd8rIrMA4JybSf4/BeBdAJPOuXnUongqi6Xv2efmsUJz8WoeUr7nK+0toV25L0X7vbVozq2FsfD2x7dSI3Le/vjWtvcJAN+sP1KVW2DhiumEwWYSstngtpltKBtJ/p8CMJX2d1WkaD+qFVZ51H3Pl8XiIZrfj5W8XX1ff7OuKvclhBm5Vq4Y6zEjzsgtmDLHfVtcvL7nS/PQscpEGQJFh2FaEcJAbmzhu3U4I7dgYpvVV3SysfETh1IXEN/J+YopE6WWot1RvT3pa+Rmlfti5b7TEMLY2XagpV8wMSUmC8ZqbDXmUoy7ENbzDYGi3VEXTh5Ozadz4eThHe03FCs7pvDdOhT9ANAsiGGRlCym1Yomrt1uEhAAWH+0daq+5Xq+GooO7yzaHWX1kIzt4RsSFP1IsJpOb5ll0wLfNljlFIoprQEQxlrFVtZwjFZ2CNCnb0Tevm+r6fSWWTY158C3rm8bNG0dHfRfESymtAaAfq3iosdsiD0UfQO0E3h8bjSLpGTautpJVBbpkn3boG3rr//pftOEr1//0/3U39ecrxDWcQ1hxTUSFhR9AzQWnu+NZpGUTFvXYhKVtq5vGzRtvfC7W6njBBd+t3UCkeZ8hbD2rtXaCiReKu/Ttxho01hXvgOOFknJtHWB/CdRaetq2jD35Qr++eHfIAD++eHfMPflSurfra6lTxRKK48tKZjlimscRI2TSou+1UCbJmLC90bTJiUD/CIbrKIgik70ZZVvXXO+dj/dnTrzdPfTO4tR16Bpr28/WA5Q82FiT6VF3yoEUWNd+d5omvVDAV1kg0UUhKa9FjHX7fKtt4q+Vpx9z1eWQZ9VbiV4vu317Qer+6aenK2eq6eenK1+DGn1+YDQU2mfvlUIosafrJrh6bl+aDB4ttdiZqPGtfLWS4dTZ/m+9dLOJhA9zHAbpZWHMIjq2w/a+8Y3IkiTnC2E8xUrlbb0LSeu+FpXvq/fluuHWlhM2vbm/bahSchl5eL6dk936rjAt1NSEFhOfMt7rWDNfaNxBWmSs4UwUTBWKi36llO5877RrML/rPyzRU/kOn1kf+qkpNNH9qfWt3BxZQXppJVbnS+L/h0/cajJDQNk5z+yEueir6+YqbR7xyphUkyvnlZhetqJXBp83AUXRwdw5mh/04SrM0f7O7po9mqG5ZpWbnW+zMIwPfIfATpxzvJWppVbXl9lp9KWPmBj4YXw6ll0fnirtyiN5XpxdKCjIt+Kxg2isZ41WPSvb/4jQHcOsgJZ08q1gQ3kMZW29K0o+tVT86ZhZTFp36J8B/timkCkTsPsaT1rsOhf7Szu7l0tg+QZ4qxOlxxbYEMgUPQNsLjR/sO3urzLQ8kPPzrot2Sj5iFV9ANVg3YGc5b1vBMs+ld7fbcuYJi1oKGmre0CBUh7KPoKfK1Rixvtl/8wgK4Wi6lrl8Mv/2Gr+0KbH77ohSAsk74Vje+DL4TwYV801/fbH9/CRsvDbONRehimpq0xPfxDo/I+fV+mF5ebfIjLq2sYn0yfODI62Ie5L1fwwY1aEq92WRt9sZhZ2bjvIsPcrNIgx0QI4cOA31iQ5lrUrpHr29ai1wmIGYq+J+2ScqXlUf9ofrkpa+NH88sY/u6eHQt/njMrQ0FzA2sEJ6YZmyH0mWaQvGhDIYTzFSsUfU80Sbm00Tvnp5ea3gpOH9m/4/wweb9pWGKR9C2EBUzq7fDNgTQ5dw/X76xslg31fzsYN9t229GbMUFtp2vkWk2oqwIUfQM07gqLxGBWbxpWWNzAoYTN+j54zk8vNQk+AFy/s4Lz00sdCzu18JNbrZELFP+2ESuVH8j1HZzNSr6VVq4ZbGyXGGy7xBTWWMd3wNOXEAb6NP1gcR1osRgkHx3sw8QrP2ganJ145QcU6wKptKWvscTeeulw6uSZtKRcRedcD0HwtOS9kHsIA32afogt974GWuRhUWnR17gANC4ITV1NYjBfYbQUvLzFuV4v78XGQxjo0/SD5jqwgn7yalBp0bdasUlT9+iB3Vt8ufXyRjQhoz/53r7UZGM/+d4+r7ZnYSHOgO7h61s3BAHTPHi0CeKsoFVefkx8+s65K865486517fzfacIYaLPn/+a/oBpLdes4/rHPz1I3WdWuS9W695aLq1YJJrJRiEkiCPVIHdL3zl3CsAdEZl1zg05506JyJTv951E6wLIO7QS8BcxTciolTBaiXNvxspVvRmD5EUv6adBYzkXnSCOVAMLS38EwFzyeSHZ1nzfMTSWWD20sjEM8v3P7+H89NKO2mDxtmH1BpMmwlnlmjb8v5Y3gnblvikAYoxgIjp8I+9IMxY+/QMNn1datn2+h3NuDMAYAPT39+fdviZ8LTHNmqsafN82NOu4at9gfAdc/5YhzmnlmtS336ynp+BKK/f11cfkBgqJWGYxh/ImFyMWon+34fOelm2f7yEiVwFcBYDh4eHOxay1wSqkzlfEXvz+36UO9L34/b/b9j4B3c2zliHOWeVWqW99HtQhhGzGRkxCGsLku1ixEP0ZAMOouW6GAEwqv98xFtaKZUidj4hpB2d932Csbh7NGrkWU/WtFiUpMzEJKd/ktk/uPv1kUPagc+44gL0iMgsAzrmZdt/nhdVShVmhc50KqQthcFYzK1mz3wsnD6cutLHjqfoGi5KUmZiENITIu1gxCdkUkbMiMisiZxvKRtp9nxdWA3gXRwdw7OCeprJjB/d0LNpCe5H7DnJp9vvWS4fR3dUizhmzkjX7HR3sw6s/2t8Urvjqj/bvaJUtq0VJykxMQmq5+E/ZKV3uHStrZXpxGQv3HjaVLdx7mEvEwPnpJRx84w949twnOPjGH1IjgjQXueZtR7Pf0cE+TJxqyaNyKj2Pira9aQni0trre2wxWa2hEJOQhrD4T6yUbkau1QCelb/TN8umZnDWKr1Evb7P8Vq117cuB3L1hDCLWQNnD2+P0om+Vc4VK8tREwrqe5FbppfQYNFe37oh5N6JEQpp+Smd6GutlaKTmFmEgsZm5Wra61s3NquVkE5ROtEH/K2V6cVl/GLy5ubCzcura/hFRhIzqxBAi1DQ2KxcTXs1dWm1ErKVUoq+L2/+dmlT8OtsPBK8+dulVLFIizvfKZrsipql94B4rFyrtNVWxDJrlZA0nHRwkYbtMDw8LHNzc0+uuA2ePfdJ5nd/vvxi0/Z/evMP+PcUkf9Wl8P/+eV/3lE7fBK5tc6WBGoWLiMWOgv7gcSCc25eRIZbyytt6WtIE/x25Rp8sivGNFuyzLAfSOxUWvQd0idpdm6tIn8Yd26Lr8uG/UBip3STszT87Gh6Bs+s8iKJabZkbGgms7EfSOyUUvR9UxBoUiu01ntSuc8sWw0xzZaMDU3qDvYDiZ3Sib7GatOkVvjVaz9OfUD86rUfb6lrseAKp53boXHZsB9I7JQueufY5c9SJ+/09fbg+rkXtl1Xw8E3/pAZe3/n0s4ifWIiltBGq+uAkCLJit4pnaWfdvNmlVsNylktuBITVimuLaDLhlSJ0ol+1kzWtHKrQTlNG8pKTGvU0mVDqkTpQjY1VrZVugLNLNuyEltoI1M2kKpQaUvfysK7ODqAM0f7mxYFOXO0v2MLroQAQxsJCZNKW/qAnYXnM8u2zMSW9I2QqlA6S78vw5LMKic20E9OSJiUztKnhRkO9JMTEh6lE/0QUu8SQkiolE70AVqYhBCSRel8+oQQQrKh6BNCSIWg6BNCSIWg6BNCSIWg6BNCSIUIPrWyc+4BgC+LbscO+A6Afym6EQbwuOKjrMfG40rnuyKyr7UweNGPHefcXFpO69jhccVHWY+Nx6WD7h1CCKkQFH1CCKkQFH17rhbdACN4XPFR1mPjcSmgT58QQioELX1CCKkQFP2ccM5dcc4dd869nvH96865+eRfb4ebt22cc0POuZk237c97lDxOK5Y+6vXOTfpnPvaOXclo06sfeZzbLH22+vJsWXpR259RtHPAefcKQB3RGS2YbuVvSLyfPJvtaMN3AEisgAgNWzM87iDpN1xJUTZXwCOi8grIrIbwJhz7kDjlzH3GZ5wbAnR9ZtzbkhE3hGRVwC8mvJ9rn1G0c+HEQBzyeeFZHuT5OIccs5JZDdZnZWM8rbHHQGpxxVzf4nIVMPmnIjcbakSbZ896dhi7bfEAIFzbgjAuylVcu0zin4+NFocKy3bEJG7IjIC4HkA78X02vkE2h53rJShv5I2L6R8FX2fZR1bzP2WtPUNAGdTvs61zyj6+dBocexp2d4keaJ/iAhvtAy8jjtWIu+vMRFJE5Ay9FnWsQGIs99EZDVx7ywkFn8jufYZRT8fZvDYPzwEYLJd5frrXAlQHXesxNZfzrnjSGK8U6zdqPvsCcfWRGz9lnAXW0U91z6j6OdA4ms8mFyQexsGXGaS/8ecc+8m36f57IIlsToONFof9ePKOu4YaHdckffXKdTaPOmcmwdwPCkvQ5+1PbZY+805dypp9ykAM/UBaKs+4+QsQgipELT0CSGkQlD0CSGkQlD0CSGkQlD0CSGkQlD0CSGkQlD0CSGkQlD0CSGkQjxVdAMI6RRJWtq9AC4BmMfjxFVDyef61P7/glr+lvlk+3nUpvT/vJ7kq57cq/53IrLqnBtDLTfKD5PfGENtduWIiPw8+b6+r3EAP6r/joiUdfUnEhi09EmVWABqeU7wOGHXcdRE+GxS/kZSPg/gCmqivUdERlqyOjb9XZLffTaZPTmT7Gch2V5NHjib+wJwuuV3COkIFH1SaRILewTAe0nRkIhcTf7tRs1y35KCOe3v6vWSafKNyb5mULP+G/fV+juEdASKPqkqvQB6nXMHEsv7bpL7ZLWeiz1jkQ7Uv2v5u7uouYXqeX3uovYgAGoPgNZVurx+h5C8Ye4dUimSRF2/Rs3yvgvgTsPXH6KWunYGwCxq2QyHUFvN6O8bV2JK/POtfzeZ7PNdEZl1zr1b34eIvJO4eF4F8PetvxNT4jMSNxR9QgipEHTvEEJIhaDoE0JIhaDoE0JIhaDoE0JIhaDoE0JIhaDoE0JIhaDoE0JIhaDoE0JIhfj/NoB0PG81BGAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "res_mat = by_concept[['rating_num','kappa_agreement']].values\n",
    "plt.scatter(res_mat[:,0], res_mat[:,1])\n",
    "plt.xlabel(\"user score\")\n",
    "plt.ylabel(\"kappa\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tobiaspytorch",
   "language": "python",
   "name": "tobiaspytorch"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}