length_analysis_en_de2.0.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from refexps import load_data, move_level, gamerun_level, get_previous #, lex_sim\n",
    "from os import listdir, path\n",
    "from itertools import combinations\n",
    "from collections import defaultdict\n",
    "import numpy as np\n",
    "from scipy import stats\n",
    "import compound_splitter\n",
    "from scipy import stats\n",
    "from nltk import word_tokenize\n",
    "from string import punctuation\n",
    "punct = list(punctuation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def exp_len(exp, language, splits, compound):\n",
    "    '''\n",
    "    Extract content words by the phrase\n",
    "    :param exp: referring expression tuple\n",
    "    :return: set of content words of the phrase\n",
    "    '''\n",
    "    exp = exp[2]\n",
    "    words = word_tokenize(exp, language)\n",
    "    if compound == True:\n",
    "        if language == 'german':\n",
    "            words_nocomp = []\n",
    "            for w in words:\n",
    "                word = compound_splitter.split_word(w, splits)\n",
    "                words_nocomp = words_nocomp + (word.split(' '))\t\n",
    "                words = words_nocomp\n",
    "    return len(words)\n",
    "\n",
    "def len_div(exp1, previous, language, splits, compound):\n",
    "    '''\n",
    "    compute dice similarity between the content words in two referring expression\n",
    "    :param exp1: referring expression tuple 1\n",
    "    :param exp2: referring expression tuple 2\n",
    "    :return: dice similarity between the content words in the expressions; undefined if they have none\n",
    "    '''        \n",
    "    len_exp = exp_len(exp1, language, splits, compound)    \n",
    "    previous_len = np.mean(previous)\n",
    "    metric = float(previous_len - len_exp)/float(previous_len + len_exp)\n",
    "    return metric if not np.isnan(metric) and metric > -1  and metric < 1 else 'undefined'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def compute_values_move(refexps_dir, success_dir, compound=True):\n",
    "    splits = compound_splitter.load_dict('de_lower.dict')\n",
    "    move_alignment = defaultdict(list)\n",
    "\n",
    "    for dialogue in listdir(refexps_dir):\n",
    "        if dialogue.startswith('FTT'):\n",
    "            language = 'german'\n",
    "        else:\n",
    "            language = 'english'\n",
    "        dialogue = path.join(refexps_dir, dialogue)\n",
    "        #load referring expressions data\n",
    "        dialogue = load_data(dialogue)\n",
    "        #move level\n",
    "        dialogue_move = move_level(dialogue)\n",
    "        #iterate over moves\n",
    "        for move in dialogue_move:\n",
    "            refs = move[1]\n",
    "            #iterate over objects\n",
    "            for obj in refs:\n",
    "                #iterate over objects\n",
    "                refobj = refs[obj]\n",
    "                #SIMILARITY OF LEXICAL CONTENT INTO\n",
    "                prev_length = []\n",
    "                speakers = set()\n",
    "                divs = []\n",
    "                divs_speakers = defaultdict(list)\n",
    "                for ref in refobj:\n",
    "                    speaker = ref[1]\n",
    "                    speakers.add(speaker)\n",
    "                    if prev_length != []:\n",
    "                        div = len_div(ref, prev_length, language, splits, compound)\n",
    "                        if div != 'undefined':\n",
    "                            divs.append(div)\n",
    "                            divs_speakers[speaker].append(div)\n",
    "                    len_exp = exp_len(ref, language, splits, compound) \n",
    "                    prev_length.append(len_exp)\n",
    "                if divs != []:\n",
    "                    alignment = np.mean(divs)\n",
    "                    move_alignment['Alignment'].append(alignment)\n",
    "                for s in speakers:\n",
    "                    if divs_speakers[speaker] != []:\n",
    "                        alignment = np.mean(divs_speakers[speaker])\n",
    "                        move_alignment['Alignment '+s].append(alignment)\n",
    "    for value in move_alignment:\n",
    "        dataset = move_alignment[value]\n",
    "        print '\\t'+ value + ': '\n",
    "        print '\\tMean: '+ str(np.mean(dataset)) +'\\tStandard deviation: '+ str(np.std(dataset))\n",
    "        print '\\tMax value: ' + str(max(dataset)) + '\\tMin value: '+ str(min(dataset))\n",
    "    return move_alignment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\tAlignment p-utts: \n",
      "\tMean: 0.125334379461\tStandard deviation: 0.274491868877\n",
      "\tMax value: 0.857142857143\tMin value: -0.714285714286\n",
      "\tAlignment: \n",
      "\tMean: 0.119894400262\tStandard deviation: 0.272796712862\n",
      "\tMax value: 0.857142857143\tMin value: -0.714285714286\n",
      "\tAlignment e-utts: \n",
      "\tMean: 0.136266563549\tStandard deviation: 0.27681838222\n",
      "\tMax value: 0.857142857143\tMin value: -0.714285714286\n",
      "\n",
      "\n",
      "\n",
      "\tAlignment p-utts: \n",
      "\tMean: 0.110288887939\tStandard deviation: 0.229964740594\n",
      "\tMax value: 0.846153846154\tMin value: -0.571428571429\n",
      "\tAlignment: \n",
      "\tMean: 0.0952154319612\tStandard deviation: 0.230235031267\n",
      "\tMax value: 0.846153846154\tMin value: -0.571428571429\n",
      "\tAlignment e-utts: \n",
      "\tMean: 0.105525507044\tStandard deviation: 0.243777371754\n",
      "\tMax value: 0.634920634921\tMin value: -0.571428571429\n",
      "\n",
      "\n",
      "\n",
      "\tAlignment p-utts: \n",
      "\tMean: 0.120511736305\tStandard deviation: 0.261142057483\n",
      "\tMax value: 0.857142857143\tMin value: -0.714285714286\n",
      "\tAlignment: \n",
      "\tMean: 0.111712305036\tStandard deviation: 0.259720559155\n",
      "\tMax value: 0.857142857143\tMin value: -0.714285714286\n",
      "\tAlignment e-utts: \n",
      "\tMean: 0.124873711937\tStandard deviation: 0.265469457462\n",
      "\tMax value: 0.857142857143\tMin value: -0.714285714286\n"
     ]
    }
   ],
   "source": [
    "de_dataset_dir = 'En_De_Dataset/De/RefExp'\n",
    "de_success_dir = 'En_De_Dataset/De/Success'\n",
    "\n",
    "en_dataset_dir = 'En_De_Dataset/En/RefExp'\n",
    "en_success_dir = 'En_De_Dataset/En/Success'\n",
    "\n",
    "all_dataset_dir = 'En_De_Dataset/All/RefExp'\n",
    "all_success_dir = 'En_De_Dataset/All/Success'\n",
    "\n",
    "results_de = compute_values_move(de_dataset_dir, de_success_dir)\n",
    "print '\\n\\n'\n",
    "results_en = compute_values_move(en_dataset_dir, en_success_dir)\n",
    "print '\\n\\n'\n",
    "results_all = compute_values_move(all_dataset_dir, all_success_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "results_de_nocs = compute_values_move(de_dataset_dir, de_success_dir, compound = False)\n",
    "print '\\n\\n'\n",
    "\n",
    "for var in results_de:\n",
    "    mannwhitneyu =  stats.mannwhitneyu(results_de[var],results_de_nocs[var], alternative='two-sided')\n",
    "    if mannwhitneyu[1] < 0.05:\n",
    "        print var\n",
    "        print str(mannwhitneyu)+'\\tSignificantly different'\n",
    "        print np.mean(results_de[var]), np.mean(results_de_nocs[var])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Differences between languages\n",
      "\n",
      "\n",
      "Differences between players\n"
     ]
    }
   ],
   "source": [
    "print 'Differences between languages'\n",
    "for var in results_all:\n",
    "    mannwhitneyu =  stats.mannwhitneyu(results_de[var],results_en[var], alternative='two-sided')\n",
    "    if mannwhitneyu[1] < 0.05:\n",
    "        print var\n",
    "        print str(mannwhitneyu)+'\\tSignificantly different'\n",
    "print '\\n'\n",
    "\n",
    "print 'Differences between players'\n",
    "mannwhitneyu =  stats.mannwhitneyu(results_de['Alignment e-utts'],results_de['Alignment p-utts'], alternative='two-sided')\n",
    "if mannwhitneyu[1] < 0.05:\n",
    "    print 'Players move alignment de'\n",
    "    print str(mannwhitneyu)+'\\tSignificantly different'\n",
    "\n",
    "mannwhitneyu =  stats.mannwhitneyu(results_en['Alignment e-utts'],results_en['Alignment p-utts'], alternative='two-sided')\n",
    "if mannwhitneyu[1] < 0.05:\n",
    "    print 'Players move alignment en'\n",
    "    print str(mannwhitneyu)+'\\tSignificantly different'\n",
    "                                                                        \n",
    "mannwhitneyu =  stats.mannwhitneyu(results_all['Alignment e-utts'],results_all['Alignment p-utts'], alternative='two-sided')\n",
    "if mannwhitneyu[1] < 0.05:\n",
    "    print 'Players move alignment all'\n",
    "    print str(mannwhitneyu)+'\\tSignificantly different'                                                             \n",
    "                                                                    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def compute_values_game(refexps_dir, success_dir, compound = True):\n",
    "    splits = compound_splitter.load_dict('de_lower.dict')\n",
    "    game_alignment = defaultdict(list)\n",
    "    for dialogue in listdir(refexps_dir):\n",
    "        if dialogue.startswith('FTT'):\n",
    "            language = 'german'\n",
    "        else:\n",
    "            language = 'english'\n",
    "        dialogue = load_data(path.join(refexps_dir, dialogue))\n",
    "        dialogue_game = gamerun_level(dialogue)\n",
    "        for obj in dialogue_game:\n",
    "            refobj = dialogue_game[obj]\n",
    "            prev_length = []\n",
    "            speakers = set()\n",
    "            divs = []\n",
    "            divs_speakers = defaultdict(list)\n",
    "            for ref in refobj:\n",
    "                speaker = ref[1]\n",
    "                speakers.add(speaker)\n",
    "                if prev_length != []:\n",
    "                    div = len_div(ref, prev_length, language, splits, compound)\n",
    "                    if div != 'undefined':\n",
    "                        divs.append(div)\n",
    "                        divs_speakers[speaker].append(div)\n",
    "                len_exp = exp_len(ref, language, splits, compound) \n",
    "                prev_length.append(len_exp)\n",
    "            if divs != []:\n",
    "                alignment = np.mean(divs)\n",
    "                game_alignment['Alignment'].append(alignment)\n",
    "            for s in speakers:\n",
    "                if divs_speakers[speaker] != []:\n",
    "                    alignment = np.mean(divs_speakers[speaker])\n",
    "                    game_alignment['Alignment '+s].append(alignment)\n",
    "    for value in game_alignment:\n",
    "        dataset = game_alignment[value]\n",
    "        print '\\t'+ value + ': '\n",
    "        print '\\tMean: '+ str(np.mean(dataset)) +'\\tStandard deviation: '+ str(np.std(dataset))\n",
    "        print '\\tMax value: ' + str(max(dataset)) + '\\tMin value: '+ str(min(dataset))\n",
    "    return game_alignment\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\tAlignment p-utts: \n",
      "\tMean: 0.110110297433\tStandard deviation: 0.237414972105\n",
      "\tMax value: 0.75\tMin value: -0.714285714286\n",
      "\tAlignment: \n",
      "\tMean: 0.110839287544\tStandard deviation: 0.2268166911\n",
      "\tMax value: 0.75\tMin value: -0.714285714286\n",
      "\tAlignment e-utts: \n",
      "\tMean: 0.102930478603\tStandard deviation: 0.208908597768\n",
      "\tMax value: 0.714285714286\tMin value: -0.714285714286\n",
      "\n",
      "\n",
      "\n",
      "\tAlignment p-utts: \n",
      "\tMean: 0.0603350300085\tStandard deviation: 0.220690275817\n",
      "\tMax value: 0.647864906832\tMin value: -0.818181818182\n",
      "\tAlignment: \n",
      "\tMean: 0.0785209763695\tStandard deviation: 0.181818114489\n",
      "\tMax value: 0.647864906832\tMin value: -0.571428571429\n",
      "\tAlignment e-utts: \n",
      "\tMean: 0.0393684392322\tStandard deviation: 0.240234900116\n",
      "\tMax value: 0.647864906832\tMin value: -0.818181818182\n",
      "\n",
      "\n",
      "\n",
      "\tAlignment p-utts: \n",
      "\tMean: 0.090713631617\tStandard deviation: 0.232313313592\n",
      "\tMax value: 0.75\tMin value: -0.818181818182\n",
      "\tAlignment: \n",
      "\tMean: 0.0978221899878\tStandard deviation: 0.21045376111\n",
      "\tMax value: 0.75\tMin value: -0.714285714286\n",
      "\tAlignment e-utts: \n",
      "\tMean: 0.0748108667157\tStandard deviation: 0.225530370179\n",
      "\tMax value: 0.714285714286\tMin value: -0.818181818182\n"
     ]
    }
   ],
   "source": [
    "de_dataset_dir = 'En_De_Dataset/De/RefExp'\n",
    "de_success_dir = 'En_De_Dataset/De/Success'\n",
    "\n",
    "en_dataset_dir = 'En_De_Dataset/En/RefExp'\n",
    "en_success_dir = 'En_De_Dataset/En/Success'\n",
    "\n",
    "all_dataset_dir = 'En_De_Dataset/All/RefExp'\n",
    "all_success_dir = 'En_De_Dataset/All/Success'\n",
    "\n",
    "results_de = compute_values_game(de_dataset_dir, de_success_dir)\n",
    "print '\\n\\n'\n",
    "results_en = compute_values_game(en_dataset_dir, en_success_dir)\n",
    "print '\\n\\n'\n",
    "results_all = compute_values_game(all_dataset_dir, all_success_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\tAlignment p-utts: \n",
      "\tMean: 0.110771195057\tStandard deviation: 0.235296558722\n",
      "\tMax value: 0.75\tMin value: -0.714285714286\n",
      "\tAlignment: \n",
      "\tMean: 0.111264108606\tStandard deviation: 0.225790136582\n",
      "\tMax value: 0.75\tMin value: -0.714285714286\n",
      "\tAlignment e-utts: \n",
      "\tMean: 0.101420266777\tStandard deviation: 0.204440782441\n",
      "\tMax value: 0.714285714286\tMin value: -0.714285714286\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "results_de_nocs = compute_values_game(de_dataset_dir, de_success_dir, compound = False)\n",
    "print '\\n\\n'\n",
    "\n",
    "for var in results_de:\n",
    "    mannwhitneyu =  stats.mannwhitneyu(results_de[var],results_de_nocs[var], alternative='two-sided')\n",
    "    if mannwhitneyu[1] < 0.05:\n",
    "        print var\n",
    "        print str(mannwhitneyu)+'\\tSignificantly different'\n",
    "        print np.mean(results_de[var]), np.mean(results_de_nocs[var])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Differences between languages\n",
      "Alignment p-utts\n",
      "MannwhitneyuResult(statistic=16838.5, pvalue=0.01043326118246455)\tSignificantly different\n",
      "Alignment\n",
      "MannwhitneyuResult(statistic=17688.5, pvalue=0.030074076978341489)\tSignificantly different\n",
      "Alignment e-utts\n",
      "MannwhitneyuResult(statistic=6783.5, pvalue=0.033790474614030862)\tSignificantly different\n",
      "\n",
      "\n",
      "Differences between players\n"
     ]
    }
   ],
   "source": [
    "print 'Differences between languages'\n",
    "for var in results_all:\n",
    "    ttest =  stats.mannwhitneyu(results_de[var],results_en[var], alternative='two-sided')\n",
    "    if ttest[1] < 0.05:\n",
    "        print var\n",
    "        print str(ttest)+'\\tSignificantly different'\n",
    "print '\\n'\n",
    "\n",
    "print 'Differences between players'\n",
    "ttest =  stats.mannwhitneyu(results_de['Alignment e-utts'],results_de['Alignment p-utts'], alternative='two-sided')\n",
    "if ttest[1] < 0.05:\n",
    "    print 'Players game alignment de'\n",
    "    print str(ttest)+'\\tSignificantly different'\n",
    "\n",
    "ttest =  stats.mannwhitneyu(results_en['Alignment e-utts'],results_en['Alignment p-utts'], alternative='two-sided')\n",
    "if ttest[1] < 0.05:\n",
    "    print 'Players game alignment en'\n",
    "    print str(ttest)+'\\tSignificantly different'\n",
    "                                                                        \n",
    "ttest =  stats.mannwhitneyu(results_all['Alignment e-utts'],results_all['Alignment p-utts'], alternative='two-sided')\n",
    "if ttest[1] < 0.05:\n",
    "    print 'Players game alignment all'\n",
    "    print str(ttest)+'\\tSignificantly different'                                                               "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}