Lecture 42 - Groupby on DataFrames.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from pandas import DataFrame,Series"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dframe = DataFrame({'k1':['X','X','Y','Y','Z'],\n",
    "                    'k2':['alpha','beta','alpha','beta','alpha'],\n",
    "                   'dataset1':np.random.randn(5),\n",
    "                   'dataset2':np.random.randn(5)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dataset1</th>\n",
       "      <th>dataset2</th>\n",
       "      <th>k1</th>\n",
       "      <th>k2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.865973</td>\n",
       "      <td>0.765498</td>\n",
       "      <td>X</td>\n",
       "      <td>alpha</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.930791</td>\n",
       "      <td>-0.121976</td>\n",
       "      <td>X</td>\n",
       "      <td>beta</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.749173</td>\n",
       "      <td>0.904476</td>\n",
       "      <td>Y</td>\n",
       "      <td>alpha</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.576850</td>\n",
       "      <td>0.271586</td>\n",
       "      <td>Y</td>\n",
       "      <td>beta</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.759003</td>\n",
       "      <td>0.331222</td>\n",
       "      <td>Z</td>\n",
       "      <td>alpha</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   dataset1  dataset2 k1     k2\n",
       "0  0.865973  0.765498  X  alpha\n",
       "1  0.930791 -0.121976  X   beta\n",
       "2 -0.749173  0.904476  Y  alpha\n",
       "3  0.576850  0.271586  Y   beta\n",
       "4 -0.759003  0.331222  Z  alpha"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<pandas.core.groupby.SeriesGroupBy object at 0x106f78b38>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "group1 = dframe['dataset1'].groupby(dframe['k1'])\n",
    "group1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "k1\n",
       "X    0.898382\n",
       "Y   -0.086161\n",
       "Z   -0.759003\n",
       "Name: dataset1, dtype: float64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "group1.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<pandas.core.groupby.SeriesGroupBy object at 0x106f78b38>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "group1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cities = np.array(['NY','LA','LA','NY','NY'])\n",
    "month = np.array(['JAN','FEB','JAN','FEB','JAN'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LA  FEB    0.930791\n",
       "    JAN   -0.749173\n",
       "NY  FEB    0.576850\n",
       "    JAN    0.053485\n",
       "Name: dataset1, dtype: float64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dframe['dataset1'].groupby([cities,month]).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dataset1</th>\n",
       "      <th>dataset2</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>k1</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>X</th>\n",
       "      <td>0.898382</td>\n",
       "      <td>0.321761</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Y</th>\n",
       "      <td>-0.086161</td>\n",
       "      <td>0.588031</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Z</th>\n",
       "      <td>-0.759003</td>\n",
       "      <td>0.331222</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    dataset1  dataset2\n",
       "k1                    \n",
       "X   0.898382  0.321761\n",
       "Y  -0.086161  0.588031\n",
       "Z  -0.759003  0.331222"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dframe.groupby('k1').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>dataset1</th>\n",
       "      <th>dataset2</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>k1</th>\n",
       "      <th>k2</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">X</th>\n",
       "      <th>alpha</th>\n",
       "      <td>0.865973</td>\n",
       "      <td>0.765498</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>beta</th>\n",
       "      <td>0.930791</td>\n",
       "      <td>-0.121976</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">Y</th>\n",
       "      <th>alpha</th>\n",
       "      <td>-0.749173</td>\n",
       "      <td>0.904476</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>beta</th>\n",
       "      <td>0.576850</td>\n",
       "      <td>0.271586</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Z</th>\n",
       "      <th>alpha</th>\n",
       "      <td>-0.759003</td>\n",
       "      <td>0.331222</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          dataset1  dataset2\n",
       "k1 k2                       \n",
       "X  alpha  0.865973  0.765498\n",
       "   beta   0.930791 -0.121976\n",
       "Y  alpha -0.749173  0.904476\n",
       "   beta   0.576850  0.271586\n",
       "Z  alpha -0.759003  0.331222"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dframe.groupby(['k1','k2']).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "k1\n",
       "X    2\n",
       "Y    2\n",
       "Z    1\n",
       "dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dframe.groupby(['k1']).size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This is the X group\n",
      "   dataset1  dataset2 k1     k2\n",
      "0  0.865973  0.765498  X  alpha\n",
      "1  0.930791 -0.121976  X   beta\n",
      "\n",
      "\n",
      "This is the Y group\n",
      "   dataset1  dataset2 k1     k2\n",
      "2 -0.749173  0.904476  Y  alpha\n",
      "3  0.576850  0.271586  Y   beta\n",
      "\n",
      "\n",
      "This is the Z group\n",
      "   dataset1  dataset2 k1     k2\n",
      "4 -0.759003  0.331222  Z  alpha\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for name,group in dframe.groupby('k1'):\n",
    "    print (\"This is the %s group\" %name)\n",
    "    print (group)\n",
    "    print ('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Key1 = X, Key2 = alpha\n",
      "   dataset1  dataset2 k1     k2\n",
      "0  0.865973  0.765498  X  alpha\n",
      "\n",
      "\n",
      "Key1 = X, Key2 = beta\n",
      "   dataset1  dataset2 k1    k2\n",
      "1  0.930791 -0.121976  X  beta\n",
      "\n",
      "\n",
      "Key1 = Y, Key2 = alpha\n",
      "   dataset1  dataset2 k1     k2\n",
      "2 -0.749173  0.904476  Y  alpha\n",
      "\n",
      "\n",
      "Key1 = Y, Key2 = beta\n",
      "   dataset1  dataset2 k1    k2\n",
      "3   0.57685  0.271586  Y  beta\n",
      "\n",
      "\n",
      "Key1 = Z, Key2 = alpha\n",
      "   dataset1  dataset2 k1     k2\n",
      "4 -0.759003  0.331222  Z  alpha\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for (k1,k2),group in dframe.groupby(['k1','k2']):\n",
    "    print (\"Key1 = %s, Key2 = %s\" %(k1,k2))\n",
    "    print (group)\n",
    "    print ('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "group_dict = dict(list(dframe.groupby('k1')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dataset1</th>\n",
       "      <th>dataset2</th>\n",
       "      <th>k1</th>\n",
       "      <th>k2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.865973</td>\n",
       "      <td>0.765498</td>\n",
       "      <td>X</td>\n",
       "      <td>alpha</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.930791</td>\n",
       "      <td>-0.121976</td>\n",
       "      <td>X</td>\n",
       "      <td>beta</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   dataset1  dataset2 k1     k2\n",
       "0  0.865973  0.765498  X  alpha\n",
       "1  0.930791 -0.121976  X   beta"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "group_dict['X']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'X':    dataset1  dataset2 k1     k2\n",
       " 0  0.865973  0.765498  X  alpha\n",
       " 1  0.930791 -0.121976  X   beta, 'Y':    dataset1  dataset2 k1     k2\n",
       " 2 -0.749173  0.904476  Y  alpha\n",
       " 3  0.576850  0.271586  Y   beta, 'Z':    dataset1  dataset2 k1     k2\n",
       " 4 -0.759003  0.331222  Z  alpha}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "group_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "group_dict_axis1 = dict(list(dframe.groupby(dframe.dtypes,axis=1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{dtype('float64'):    dataset1  dataset2\n",
       " 0  0.865973  0.765498\n",
       " 1  0.930791 -0.121976\n",
       " 2 -0.749173  0.904476\n",
       " 3  0.576850  0.271586\n",
       " 4 -0.759003  0.331222, dtype('O'):   k1     k2\n",
       " 0  X  alpha\n",
       " 1  X   beta\n",
       " 2  Y  alpha\n",
       " 3  Y   beta\n",
       " 4  Z  alpha}"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "group_dict_axis1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dataset2_group = dframe.groupby(['k1','k2'])[['dataset2']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<pandas.core.groupby.DataFrameGroupBy object at 0x106fe6eb8>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset2_group"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>dataset2</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>k1</th>\n",
       "      <th>k2</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">X</th>\n",
       "      <th>alpha</th>\n",
       "      <td>0.765498</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>beta</th>\n",
       "      <td>-0.121976</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">Y</th>\n",
       "      <th>alpha</th>\n",
       "      <td>0.904476</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>beta</th>\n",
       "      <td>0.271586</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Z</th>\n",
       "      <th>alpha</th>\n",
       "      <td>0.331222</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          dataset2\n",
       "k1 k2             \n",
       "X  alpha  0.765498\n",
       "   beta  -0.121976\n",
       "Y  alpha  0.904476\n",
       "   beta   0.271586\n",
       "Z  alpha  0.331222"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset2_group.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}