clustering_by_time.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# This script calculate clusters for each time period"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import csv\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "from sklearn.cluster import KMeans\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Initializing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Calculated via elbow_rule_for_times.ipynb\n",
    "cluster_num = 4\n",
    "border_times = [7,14,28,56,200]\n",
    "used_names = ['activity_quantity','med_ml_num','buynum', 'grind','quests_speed']\n",
    "f = open(\"clusters_timed.csv\",\"wb\")\n",
    "writer = csv.writer(f, delimiter = '|')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Working with every time period"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "For days = 7\n",
      "['activity_quantity', 'med_ml_num', 'buynum', 'grind', 'quests_speed']\n",
      "[ 3.12976057  0.06274624  3.89386     6.75372659  2.33786097]\n",
      "    8.9000      0.0080    102.4518    886.2520      9.1061\n",
      "[  3.91376451e+00   1.05444997e-01   1.42123825e+01   5.83821144e+02\n",
      "   4.69803759e+00]\n",
      "   10.3779      0.0134    373.9438  76611.4324     17.2895\n",
      "[  1.00000000e+00   0.00000000e+00   0.00000000e+00   1.25160000e+04\n",
      "   0.00000000e+00]\n",
      "    4.8851      0.0000      0.0000  1642401.4395      1.0000\n",
      "[  3.08333333e+00   8.58333333e-02   7.81083333e+00   2.28971667e+03\n",
      "   4.58416667e+00]\n",
      "    8.8124      0.0109    205.5118  300466.1193     16.8947\n",
      "For days = 14\n",
      "['activity_quantity', 'med_ml_num', 'buynum', 'grind', 'quests_speed']\n",
      "[ 5.0680408   0.05488972  3.35814545  6.10243227  1.87118588]\n",
      "   24.8794      0.0059     75.9444    773.2608      6.3813\n",
      "[  5.02380952e+00   6.29534817e-02   6.90655872e+00   1.88532854e+03\n",
      "   2.96138052e+00]\n",
      "   24.6971      0.0067    156.1917  238896.6614      9.5165\n",
      "[  1.00000000e+00   0.00000000e+00   0.00000000e+00   1.19770000e+04\n",
      "   0.00000000e+00]\n",
      "    8.1198      0.0000      0.0000  1517648.1194      1.0000\n",
      "[  6.84871795e+00   9.68566334e-02   1.05076322e+01   4.38688526e+02\n",
      "   3.88369171e+00]\n",
      "   32.2154      0.0104    237.6300  55587.7779     12.1690\n",
      "For days = 28\n",
      "['activity_quantity', 'med_ml_num', 'buynum', 'grind', 'quests_speed']\n",
      "[ 7.71522349  0.04778265  3.05579515  6.24372021  1.59173983]\n",
      "   66.2710      0.0046     62.0826    796.6816      4.9325\n",
      "[  9.00000000e+00   1.80555556e-01   2.63888889e-01   1.24737222e+04\n",
      "   2.19444444e+00]\n",
      "   76.6407      0.0174      5.3613  1591612.7494      6.5229\n",
      "[  5.73076923e+00   6.54665871e-02   1.49553889e+01   1.72052581e+03\n",
      "   3.18297511e+00]\n",
      "   50.2541      0.0063    303.8390  219534.3757      9.1315\n",
      "[  1.11862745e+01   8.36505274e-02   9.43128343e+00   4.35558417e+02\n",
      "   3.34344115e+00]\n",
      "   94.2866      0.0081    191.6093  55576.0596      9.5550\n",
      "For days = 56\n",
      "['activity_quantity', 'med_ml_num', 'buynum', 'grind', 'quests_speed']\n",
      "[ 10.800881     0.04309896   2.85186237   5.7083882    1.46645397]\n",
      "  157.5309      0.0039     55.2963    608.0016      4.3083\n",
      "[  1.50000000e+01   1.72413793e-02   3.27586207e-01   1.26443448e+04\n",
      "   2.41379310e-01]\n",
      "  217.2200      0.0015      6.3517  1346751.7845      1.1648\n",
      "[  1.78579545e+01   7.65240224e-02   1.11608053e+01   3.97617641e+02\n",
      "   3.22658718e+00]\n",
      "  257.8449      0.0068    216.4028  42350.3372      8.8248\n",
      "[  4.89189189e+00   6.79436424e-02   1.46020456e+01   1.62878390e+03\n",
      "   3.07554277e+00]\n",
      "   73.5366      0.0061    283.1268  173482.1102      8.4372\n",
      "For days = 200\n",
      "['activity_quantity', 'med_ml_num', 'buynum', 'grind', 'quests_speed']\n",
      "[ 14.34941627   0.03976582   2.63320205   5.36499713   1.39237357]\n",
      "  368.5639      0.0034     48.2881    452.9407      3.9699\n",
      "[  6.43137255e+00   4.79734090e-02   1.68130253e+01   1.80732898e+03\n",
      "   3.15017356e+00]\n",
      "  167.3966      0.0041    308.3202  152584.0162      8.3505\n",
      "[  1.00000000e+00   0.00000000e+00   0.00000000e+00   1.14380000e+04\n",
      "   0.00000000e+00]\n",
      "   29.4062      0.0000      0.0000  965654.8415      0.5000\n",
      "[  2.55841270e+01   7.15454554e-02   1.18837105e+01   3.97114807e+02\n",
      "   3.27475602e+00]\n",
      "  653.9950      0.0061    217.9255  33526.4763      8.6609\n"
     ]
    }
   ],
   "source": [
    "for days in border_times:\n",
    "    scaling_params = [] # This is pairs of (expected value, variance)\n",
    "    # this function make each param standart normally distributed\n",
    "    def scaling(df):\n",
    "        for x in used_names:\n",
    "            exp_val = df[x].median()\n",
    "            variance = df[x].var()\n",
    "            scaling_params.append((exp_val,variance**0.5))\n",
    "            df[x].apply(lambda x: (x-exp_val)/variance)\n",
    "            q = df[x].quantile([0.02,0.92])\n",
    "            def quan(x):\n",
    "                if x<q[0.02] or x>q[0.92]:\n",
    "                    return q[0.02] if x<q[0.02] else q[0.92]\n",
    "                return x\n",
    "            df[x].apply(quan)\n",
    "        return df\n",
    "    # this function return converts values back\n",
    "    def rescaling(values):\n",
    "        #print (scaling_params,values)\n",
    "        return [x[0][0]+x[0][1]*x[1] for x in zip(scaling_params,values)]\n",
    "    df = pd.read_csv('preprocessed_%d.csv'%days, sep = '|', low_memory = False)\n",
    "    df = scaling(df)\n",
    "    writer.writerow([days,':']+['']*3)\n",
    "    # search clusters\n",
    "    kmeans = KMeans(n_clusters = cluster_num)\n",
    "    kmeans.fit(df.ix[:,used_names])\n",
    "    best = kmeans\n",
    "    for x in xrange(20):\n",
    "        kmeans= KMeans(n_clusters = cluster_num)\n",
    "        kmeans.fit(df.ix[:,used_names])\n",
    "        if kmeans.inertia_ < best.inertia_:\n",
    "            best = kmeans\n",
    "    kmeans = best\n",
    "    prediction = kmeans.predict(df.ix[:,used_names])\n",
    "    # output of cluster centroids\n",
    "    writer.writerow(['cluster_id', 'user_count'] + used_names)\n",
    "    print \"For days = %d\"%days\n",
    "    print used_names\n",
    "    ind = 0\n",
    "    for x in kmeans.cluster_centers_:\n",
    "        print x\n",
    "        res = rescaling(x)\n",
    "        writer.writerow([ind, len([x for x in prediction if ind == x])]+res)\n",
    "        print '  '.join(\"{:10.4f}\".format(i) for i in res)\n",
    "        ind +=1\n",
    "    # output of clusters\n",
    "    with open('cluster_elements_%d.csv'%days,'wb') as clusters_file:\n",
    "        wr = csv.writer(clusters_file, delimiter='|',\n",
    "                                quotechar='\"', quoting=csv.QUOTE_MINIMAL)\n",
    "        wr.writerow(['id','cluster_index'])\n",
    "        for i in xrange(len(prediction)):\n",
    "            wr.writerow([df['id'].ix[i],prediction[i]])\n",
    "        clusters_file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}