DataVisualization.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c05f3477",
   "metadata": {},
   "source": [
    "# Imports & Function Definitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "abd5bfe9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# imports\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import plotly.express as px\n",
    "import plotly.io as pio\n",
    "import kaleido as kal\n",
    "import seaborn as sns\n",
    "\n",
    "from matplotlib.lines import Line2D \n",
    "from io import StringIO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2df398c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# method to get the unique values for a list\n",
    "def unique(lst, _print = False):\n",
    "    unique_list = []\n",
    "    \n",
    "    for elem in lst:\n",
    "        elem = str(elem)\n",
    "        for x in elem.split(\"; \"):\n",
    "            x_s = x.strip()\n",
    "            if x_s not in unique_list:\n",
    "                unique_list.append(x_s)\n",
    "            \n",
    "    if _print:\n",
    "        print(unique_list)\n",
    "    \n",
    "    unique_list.sort()\n",
    "    \n",
    "    return unique_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "da655593",
   "metadata": {},
   "outputs": [],
   "source": [
    "# method to map the terms in a list to an index through a dictionary\n",
    "def map_idx(lst):\n",
    "    d = dict() \n",
    "    i = 0\n",
    "    \n",
    "    for x in lst:\n",
    "        d[x] = i\n",
    "        i = i+1\n",
    "        \n",
    "    return d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d211d014",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# creates a correlation by counting the co-occurrences of the values in the two given columns\n",
    "def relate_columns(col_name, row_name):\n",
    "    columns = unique(data.loc[: , col_name])\n",
    "    rows = unique(data.loc[: , row_name])\n",
    "\n",
    "    column_idx = map_idx(columns)\n",
    "    row_idx = map_idx(rows)\n",
    "\n",
    "    corr = [[0 for x in range(len(columns))] for y in range(len(rows))] \n",
    "\n",
    "    for idx, row in data.iterrows():\n",
    "        _col = str(row[col_name])\n",
    "        _row = str(row[row_name])\n",
    "        \n",
    "        for c in _col.split(\"; \"):\n",
    "            for r in _row.split(\"; \"):\n",
    "                c_s = c.strip()\n",
    "                r_s = r.strip()\n",
    "                \n",
    "                val = corr[row_idx[r_s]][column_idx[c_s]]\n",
    "                val = val + 1\n",
    "                corr[row_idx[r_s]][column_idx[c_s]] = val\n",
    "\n",
    "    df_data = np.array(corr)\n",
    "    df = pd.DataFrame(data=df_data, index=rows, columns=columns)\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "63fe5b6d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# counts the occurrences of the entities for the given column\n",
    "def count_entries(name, thresh=-1, asc=True):\n",
    "    values = unique(data.loc[: , name])\n",
    "    d = dict()\n",
    "    \n",
    "    for idx, row in data.iterrows():\n",
    "        _row = str(row[name])\n",
    "        \n",
    "        for e in _row.split(\"; \"):\n",
    "            e_s = e.strip()\n",
    "            d[e_s]= 1 if e_s not in d else d[e_s]+1\n",
    "    \n",
    "    if thresh >= 0:\n",
    "        new_d = dict()\n",
    "        for k in d.keys():\n",
    "            if d[k] >= thresh:\n",
    "                new_d[k] = d[k]\n",
    "        d = new_d\n",
    "    \n",
    "    keys = list(d.keys())\n",
    "    values = np.array(list(d.values()))\n",
    "    if asc:\n",
    "        sorted_value_index = np.argsort(values)\n",
    "    else:\n",
    "        sorted_value_index = np.argsort(-values)  \n",
    "    sorted_dict = {keys[i]: values[i] for i in sorted_value_index}\n",
    "\n",
    "    return sorted_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c98cde77",
   "metadata": {},
   "outputs": [],
   "source": [
    "# counts the occurrences of the entities per row for the given column\n",
    "def count_row_entries(name, thresh = -1, asc=True):\n",
    "    values = unique(data.loc[: , name])\n",
    "    d = dict()\n",
    "    \n",
    "    for idx, row in data.iterrows():\n",
    "        _cell = str(row[name])\n",
    "        _row = str(row[\"Key\"])\n",
    "        val = len(_cell.split(\"; \"))\n",
    "        d[_row] = val\n",
    "    \n",
    "    if thresh >= 0:\n",
    "        new_d = dict()\n",
    "        for k in d.keys():\n",
    "            if d[k] >= thresh:\n",
    "                new_d[k] = d[k]\n",
    "        d = new_d\n",
    "    \n",
    "    keys = list(d.keys())\n",
    "    values = list(d.values())\n",
    "    if asc:\n",
    "        sorted_value_index = np.argsort(values)\n",
    "    else:\n",
    "        sorted_value_index = np.argsort(-values)    \n",
    "    sorted_dict = {keys[i]: values[i] for i in sorted_value_index}\n",
    "\n",
    "    return sorted_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e0cc37df",
   "metadata": {},
   "source": [
    "# Data Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4a92c36d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# read csv file into variable\n",
    "data = pd.read_csv('./Data/Final Publications.csv')\n",
    "data = data.fillna(\"None\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2a367d4f",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Relate two columns by counting the amount of publications wit overlapping values\n",
    "# The first named column sets the x-axis, the second the y-axis\n",
    "relate_columns(\"Use Case\", \"Domain\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1d7ef763",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get the highest related value and its corresponding value from the second column for the values in the first column\n",
    "df = relate_columns(\"Use Case\", \"CSK Source\")\n",
    "print(df.max())\n",
    "print(df.idxmax())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "859d5b17",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Count entries in the given column\n",
    "count_entries('Evaluation DS')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01ce860e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# look for overlaps in data from Zech2019 and our publications\n",
    "data_zech = pd.read_csv('./Data/Publications Zech2019.csv')\n",
    "overlap = []\n",
    "\n",
    "for idx, row in data_zech.iterrows():\n",
    "    entry_zech = str(row[\"Title\"])\n",
    "    for idx2, row2 in data.iterrows():\n",
    "        entry_us = str(row2[\"Title\"])\n",
    "        if entry_us.casefold() == entry_zech.casefold():\n",
    "            overlap.append(entry_zech)\n",
    "            \n",
    "print(overlap)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3af20f3d",
   "metadata": {},
   "source": [
    "# Visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "62ec5fee",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Duplicates Heat Map\n",
    "data_dup = pd.read_csv('./Data/Duplicates.csv')\n",
    "data_dup = data_dup.set_index('Unnamed: 0')\n",
    "fig = px.imshow(data_dup,\n",
    "                text_auto=True,\n",
    "                aspect=\"auto\",\n",
    "                labels=dict(x=\"Source\", y=\"Source\", color=\"# Duplicates\"),\n",
    "                color_continuous_scale=px.colors.sequential.RdBu,\n",
    "                height=700, width=900)\n",
    "fig.update_layout({\n",
    "    \"plot_bgcolor\": \"rgba(1, 0, 0, 0)\",\n",
    "    #\"paper_bgcolor\": \"rgba(1, 0, 0, 0)\"\n",
    "})\n",
    "#fig.show()\n",
    "pio.write_image(fig, './Images/Duplicates.pdf', format='pdf', engine='kaleido')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7aeae1e7",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# CSK Source Usage bar chart (vertical)\n",
    "\n",
    "lst = [0, 0, 0, 0, 0, 0]\n",
    "for k, v in count_row_entries('CSK Source').items():\n",
    "    pos = v\n",
    "    if v == 1:\n",
    "        row = data.loc[data['Key'] == k]\n",
    "        if row['CSK Source'].iloc[0] == 'Not mentioned':\n",
    "            pos = 0\n",
    "        else:\n",
    "            pos = 1\n",
    "    lst[pos] += 1\n",
    "\n",
    "df = pd.DataFrame(lst)\n",
    "ax = df.plot(kind='bar', stacked=False, ylabel='# Publications', xlabel='Used Sources', rot=0, legend=False)\n",
    "ax.bar_label(ax.containers[0])\n",
    "fig = ax.get_figure()\n",
    "fig.savefig('./Images/CSK Source Usage.pdf', dpi=300, format='pdf', bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "89d2a52c",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# CSK Source Type bar chart (vertical)\n",
    "\n",
    "s = StringIO(\"\"\"Type;Sources;Publications\n",
    "Human;3;18\n",
    "Unstructured;10;17\n",
    "Semi-Structured;6;11\n",
    "Structured;10;26\"\"\")\n",
    "\n",
    "df = pd.read_csv(s, index_col=0, delimiter=';', skipinitialspace=True)\n",
    "ax = df.plot(kind='bar', stacked=False, ylabel='Amount', xlabel='Type', rot=0)\n",
    "for container in ax.containers:\n",
    "    ax.bar_label(container)\n",
    "fig = ax.get_figure()\n",
    "fig.savefig('./Images/CSK Source Type.pdf', dpi=300, format='pdf', bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9cceea13",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Application Domains\n",
    "\n",
    "domains = count_entries('Domain', asc=False)\n",
    "df = pd.DataFrame(domains.values(), index=domains.keys())\n",
    "ax = df.plot(kind='bar', stacked=False, ylabel='# Publications', xlabel='Domain', rot=0, legend=False)\n",
    "ax.bar_label(ax.containers[0])\n",
    "fig = ax.get_figure()\n",
    "fig.savefig('./Images/Domains.pdf', dpi=300, format='pdf', bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7523a729",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Use Cases\n",
    "\n",
    "s = StringIO(\"\"\"use case;Focus;Proof-of-Concept;Neither\n",
    "Object Localization;3;5;0\n",
    "Object Delivery;2;5;0\n",
    "Environment Exploration;4;1;0\n",
    "Tool Substitution;5;0;0\n",
    "Object Recognition;3;1;0\n",
    "Intention Inference;2;1;0\n",
    "Pick & Place;0;3;0\n",
    "Cooking;0;2;0\n",
    "Navigation;1;1;0\n",
    "Table Setting;1;1;0\n",
    "Tidy Up;2;0;0\n",
    "Warehousing;0;2;0\n",
    "Hole Digging;0;1;0\n",
    "Location Detection;0;1;0\n",
    "Reminiscence Therapy;1;0;0\n",
    "None;0;0;4\"\"\")\n",
    "\n",
    "df = pd.read_csv(s, index_col=0, delimiter=';', skipinitialspace=True)\n",
    "dfp = df.pivot_table(values=['Focus', 'Proof-of-Concept', 'Neither'], index='use case', aggfunc=np.sum, sort=False)\n",
    "fig = dfp.plot(kind='barh', stacked=True, ylabel='', xlabel='# Publications', rot=0).get_figure()\n",
    "\n",
    "ax = fig.gca()\n",
    "for bar in ax.patches:\n",
    "    width = bar.get_width()\n",
    "    height = bar.get_height()\n",
    "    x = bar.get_x()\n",
    "    y = bar.get_y()\n",
    "    if width > 0:\n",
    "        ax.text(x + width/2, y + height/2, f'{width:.0f}', ha='center', va='center', color='black', fontsize=10)\n",
    "\n",
    "fig.savefig('./Images/Use Cases.pdf', dpi=300, format='pdf', bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "557ddf4d",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Year\n",
    "\n",
    "years = count_entries('Year')\n",
    "years.update({\"2017\": 0}) \n",
    "df = pd.DataFrame(years.values(), index=years.keys())\n",
    "df.sort_index(inplace=True)\n",
    "\n",
    "ax = df.plot(kind='bar', stacked=False, ylabel='# Publications', xlabel='Year', rot=0, legend=False)\n",
    "ax.bar_label(ax.containers[0])\n",
    "fig = ax.get_figure()\n",
    "fig.savefig('./Images/Year.pdf', dpi=300, format='pdf', bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "097e2fd9",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Venue\n",
    "\n",
    "venue = count_entries('Type', asc=False)\n",
    "df = pd.DataFrame(venue.values(), index=venue.keys())\n",
    "ax = df.plot(kind='bar', stacked=False, ylabel='# Publications', xlabel='Venue Type', rot=0, legend=False)\n",
    "ax.bar_label(ax.containers[0])\n",
    "fig = ax.get_figure()\n",
    "fig.savefig('./Images/Venue Type.pdf', dpi=300, format='pdf', bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4e2918c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Questions\n",
    "\n",
    "s = StringIO(\"\"\"use case;amount;cat\n",
    "What is the expected location for an object?;22;a) Objects\n",
    "What affordances does an object have?;17;a) Objects\n",
    "Which tools can be used for a certain task?;11;a) Objects\n",
    "Which objects are similar to the given object?;9;a) Objects\n",
    "How can I interact with an object / container / etc.?;8;a) Objects\n",
    "Can I accomplish the given task or do I need help?;7;b) Interaction\n",
    "What are the physical properties of an object (e.g. size, shape, color)?;7;a) Objects\n",
    "Where to place objects (on a table)?;7;a) Objects\n",
    "How can an object be transported / grasped?;6;a) Objects\n",
    "How can I react to an incomplete command by a human?;4;b) Interaction\n",
    "What are the spatial relations of this object?;4;a) Objects\n",
    "What materials make up the object?;4;a) Objects\n",
    "Which objects in the environment need to be avoided?;4;a) Objects\n",
    "What are the intentions a human could have with a certain object?;3;b) Interaction\n",
    "What is the outcome of my current action?;3;c) Causality\n",
    "Where are specific humans located?;3;b) Interaction\n",
    "Does my new knowledge contradict my knowledge base?;2;c) Causality\n",
    "What activity / event do I perceive?;2;c) Causality\n",
    "What is the location where certain objects are currently located?;2;a) Objects\n",
    "What parts does the object consist of?;2;a) Objects\n",
    "Which brand produced the object?;2;a) Objects\n",
    "What aspects of my environment are changing?;1;c) Causality\n",
    "What is the (current) functional state of the object?;1;a) Objects\n",
    "What is the (shortest) distance to my current goal?;1;c) Causality\n",
    "What is the sentiment of a concept (positive / negative)?;1;c) Causality\"\"\")\n",
    "\n",
    "colors = {'a) Objects':'tab:blue', 'b) Interaction':'tab:orange', 'c) Causality':'tab:green'}\n",
    "df = pd.read_csv(s, index_col=0, delimiter=';', skipinitialspace=True)\n",
    "\n",
    "fig = plt.figure() # Create matplotlib figure\n",
    "\n",
    "ax = fig.add_subplot(111) # Create matplotlib axes\n",
    "width = 0.5\n",
    "\n",
    "df.amount.plot(kind='barh', color=df['cat'].map(colors), ax=ax, width=width, position=0.5, rot=0)\n",
    "ax.bar_label(ax.containers[0], padding=3)\n",
    "\n",
    "#add legend\n",
    "handles = [Line2D([0], [0], marker='o', color='w', markerfacecolor=v, label=k, markersize=8) for k, v in colors.items()]\n",
    "ax.legend(title='Category', handles=handles, bbox_to_anchor=(0.75, 0.75), loc='center')\n",
    "\n",
    "ax.set_xlabel('# Publications providing an answer')\n",
    "ax.set_ylabel('Questions')\n",
    "plt.savefig('./Images/Questions.pdf', dpi=300, format='pdf', bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dcaec3d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Evaluation Setting\n",
    "\n",
    "s = StringIO(\"\"\"Setting;Approaches;Simulated;Real-World\n",
    "None;2;0;0\n",
    "Questionnaire;1;0;0\n",
    "Motivating Example;5;0;0\n",
    "Model Evaluation;7;0;0\n",
    "Case Study;0;6;7\n",
    "Experiment;0;20;12\"\"\")\n",
    "\n",
    "df = pd.read_csv(s, index_col=0, delimiter=';', skipinitialspace=True)\n",
    "dfp = df.pivot_table(values=['Approaches', 'Simulated', 'Real-World'], index='Setting', aggfunc=np.sum, sort=False)\n",
    "fig = dfp.plot(kind='barh', stacked=True, ylabel='', xlabel='# Publications', rot=0).get_figure()\n",
    "\n",
    "ax = fig.gca()\n",
    "for bar in ax.patches:\n",
    "    width = bar.get_width()\n",
    "    height = bar.get_height()\n",
    "    x = bar.get_x()\n",
    "    y = bar.get_y()\n",
    "    if width > 0:\n",
    "        ax.text(x + width/2, y + height/2, f'{width:.0f}', ha='center', va='center', color='black', fontsize=10)\n",
    "\n",
    "fig.savefig('./Images/Evaluation Setting.pdf', dpi=300, format='pdf', bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a718bc7f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Main CSK Sources\n",
    "\n",
    "sources = count_entries('CSK Source', asc=False)\n",
    "small_sources = [k for k, v in sources.items() if v == 1]\n",
    "for s in small_sources:\n",
    "    del sources[s]\n",
    "sources['Others'] = len(small_sources)\n",
    "\n",
    "df = pd.DataFrame(sources.values(), index=sources.keys())\n",
    "ax = df.plot(kind='bar', stacked=False, ylabel='# Publications', xlabel='', rot=90, legend=False)\n",
    "ax.bar_label(ax.containers[0])\n",
    "fig = ax.get_figure()\n",
    "ax.set_xticklabels(df.index, rotation=45, ha='right')\n",
    "fig.savefig('./Images/CSK Main Sources.pdf', dpi=300, format='pdf', bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d25142cd",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}