IMDB_AttackDemo.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/malzantot/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import data_utils\n",
    "import glove_utils\n",
    "import models\n",
    "import display_utils\n",
    "from goog_lm import LM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import lm_data_utils\n",
    "import lm_utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(1001)\n",
    "tf.set_random_seed(1001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "VOCAB_SIZE  = 50000\n",
    "with open('aux_files/dataset_%d.pkl' %VOCAB_SIZE, 'rb') as f:\n",
    "    dataset = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc_len = [len(dataset.test_seqs2[i]) for i in \n",
    "           range(len(dataset.test_seqs2))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "dist_mat = np.load('aux_files/dist_counter_%d.npy' %VOCAB_SIZE)\n",
    "# Prevent returning 0 as most similar word because it is not part of the dictionary\n",
    "dist_mat[0,:] = 100000\n",
    "dist_mat[:,0] = 100000\n",
    "\n",
    "skip_list = np.load('aux_files/missed_embeddings_counter_%d.npy' %VOCAB_SIZE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Demonstrating how we find the most similar words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Closest to `later` are:\n",
      " --  subsequent   0.18323109771400015\n",
      " --  subsequently   0.1867195991340007\n",
      " --  afterward   0.2509214012219996\n",
      " --  afterwards   0.2576958961479996\n",
      " --  thereafter   0.2741981096589998\n",
      " --  trailing   0.3368002712810001\n",
      " --  after   0.34520261237799876\n",
      " --  then   0.36472839338299834\n",
      " --  posterior   0.4310855888389997\n",
      " --  following   0.4833073676040003\n",
      "----\n",
      "Closest to `takes` are:\n",
      " --  pick   0.31130546563200046\n",
      " --  taking   0.42471158462800007\n",
      " --  picked   0.48527412495900113\n",
      "----\n",
      "Closest to `instead` are:\n",
      " --  conversely   0.30340380498499964\n",
      " --  however   0.3475382865829997\n",
      " --  alternatively   0.39540487543000014\n",
      " --  alternately   0.4439627395600003\n",
      " --  nevertheless   0.477163975792001\n",
      "----\n",
      "Closest to `seem` are:\n",
      " --  seems   0.007052995653001215\n",
      " --  appears   0.32837244735200044\n",
      " --  looks   0.33534638306400066\n",
      " --  transpires   0.456207185493001\n",
      "----\n",
      "Closest to `beautiful` are:\n",
      " --  gorgeous   0.019236443661999614\n",
      " --  wonderful   0.10149643378299977\n",
      " --  splendid   0.10299021060599989\n",
      " --  handsome   0.11803810151499938\n",
      " --  resplendent   0.11808701124700005\n",
      " --  wondrous   0.12150066282500016\n",
      " --  marvelous   0.12519975539099892\n",
      " --  marvellous   0.1283983423189996\n",
      " --  fantastic   0.1362662712549989\n",
      " --  sumptuous   0.14140581277100006\n",
      " --  magnificent   0.1463668716009996\n",
      " --  terrific   0.15607668417800036\n",
      " --  lovely   0.1600076271229991\n",
      " --  ravishing   0.17404625231200033\n",
      " --  sublime   0.17909557696099943\n",
      " --  exquisite   0.1881071417279998\n",
      " --  fabulous   0.20363493095600038\n",
      " --  delightful   0.20468405530899902\n",
      " --  superb   0.21660537682999959\n",
      " --  excellent   0.22435712285300036\n",
      "----\n"
     ]
    }
   ],
   "source": [
    "for i in range(300, 305):\n",
    "    src_word = i\n",
    "    nearest, nearest_dist = glove_utils.pick_most_similar_words(src_word, dist_mat,20, 0.5)\n",
    "        \n",
    "    print('Closest to `%s` are:' %(dataset.inv_dict[src_word]))\n",
    "    for w_id, w_dist in zip(nearest, nearest_dist):\n",
    "          print(' -- ', dataset.inv_dict[w_id], ' ', w_dist)\n",
    "\n",
    "    print('----')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preparing the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_len = 250\n",
    "train_x = pad_sequences(dataset.train_seqs2, maxlen=max_len, padding='post')\n",
    "train_y = np.array(dataset.train_y)\n",
    "test_x = pad_sequences(dataset.test_seqs2, maxlen=max_len, padding='post')\n",
    "test_y = np.array(dataset.test_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading the sentiment analysis model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./models/imdb_model\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "if tf.get_default_session():\n",
    "    sess.close()\n",
    "sess = tf.Session()\n",
    "batch_size = 1\n",
    "lstm_size = 128\n",
    "#max_len =  100\n",
    "\n",
    "with tf.variable_scope('imdb', reuse=False):\n",
    "    model = models.SentimentModel(batch_size=batch_size,\n",
    "                           lstm_size = lstm_size,\n",
    "                           max_len = max_len,\n",
    "                           embeddings_dim=300, vocab_size=dist_mat.shape[1],is_train = False)\n",
    "saver = tf.train.Saver()\n",
    "saver.restore(sess, './models/imdb_model')\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading the Google Language model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LM vocab loading done\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Recovering graph.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Recovering Graph goog_lm/graph-2016-09-10.pbtxt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Recovering checkpoint goog_lm/ckpt-*\n"
     ]
    }
   ],
   "source": [
    "goog_lm = LM()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "#### demonstrating the GoogLM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Closest to `play` are ['playing', 'gaming', 'games', 'toy', 'playback', 'game', 'plaything', 'cheek', 'gambling', 'toys', 'toying', 'replay', 'stake', 'plays', 'jeu', 'gamble', 'staking', 'reproduction', 'casino', 'sets']\n"
     ]
    }
   ],
   "source": [
    "src_word = dataset.dict['play']\n",
    "nearest, nearest_dist = glove_utils.pick_most_similar_words(src_word, dist_mat,20)\n",
    "nearest_w = [dataset.inv_dict[x] for x in nearest]\n",
    "print('Closest to `%s` are %s' %(dataset.inv_dict[src_word], nearest_w))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "most probable is  game\n"
     ]
    }
   ],
   "source": [
    "prefix = 'is'\n",
    "suffix = 'with'\n",
    "lm_preds = goog_lm.get_words_probs(prefix, nearest_w, suffix)\n",
    "print('most probable is ', nearest_w[np.argmax(lm_preds)])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Try Attack"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from attacks import GeneticAtack"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Main Attack "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "pop_size = 60\n",
    "n1 = 8\n",
    "\n",
    "with tf.variable_scope('imdb', reuse=True):\n",
    "    batch_model = models.SentimentModel(batch_size=pop_size,\n",
    "                           lstm_size = lstm_size,\n",
    "                           max_len = max_len,\n",
    "                           embeddings_dim=300, vocab_size=dist_mat.shape[1],is_train = False)\n",
    "    \n",
    "with tf.variable_scope('imdb', reuse=True):\n",
    "    neighbour_model = models.SentimentModel(batch_size=n1,\n",
    "                           lstm_size = lstm_size,\n",
    "                           max_len = max_len,\n",
    "                           embeddings_dim=300, vocab_size=dist_mat.shape[1],is_train = False)\n",
    "ga_atttack = GeneticAtack(sess, model, batch_model, neighbour_model, dataset, dist_mat, \n",
    "                                  skip_list,\n",
    "                                  goog_lm, max_iters=30, \n",
    "                                   pop_size=pop_size,\n",
    "                                  \n",
    "                                  n1 = n1,\n",
    "                                  n2 = 4,\n",
    "                                 use_lm = True, use_suffix=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shortest sentence in our test set is 18 words\n",
      "******  1  ********\n",
      "\t\t 0  --  0.16752617\n",
      "\t\t 1  --  0.33954847\n",
      "\t\t 2  --  0.62575793\n",
      "1 - 4 changed.\n",
      "--------------------------\n",
      "******  2  ********\n",
      "\t\t 0  --  0.081095785\n",
      "\t\t 1  --  0.23031871\n",
      "\t\t 2  --  0.49455175\n",
      "\t\t 3  --  0.8090076\n",
      "3 - 6 changed.\n",
      "--------------------------\n",
      "******  3  ********\n",
      "\t\t 0  --  0.20659587\n",
      "\t\t 1  --  0.29161933\n",
      "\t\t 2  --  0.34426233\n",
      "\t\t 3  --  0.58601284\n",
      "7 - 5 changed.\n",
      "--------------------------\n",
      "******  4  ********\n",
      "\t\t 0  --  0.00016315776\n",
      "\t\t 1  --  0.00029146814\n",
      "\t\t 2  --  0.00045838807\n",
      "\t\t 3  --  0.0006008647\n",
      "\t\t 4  --  0.0008673914\n",
      "\t\t 5  --  0.001375048\n",
      "\t\t 6  --  0.0021017992\n",
      "\t\t 7  --  0.0031949652\n",
      "\t\t 8  --  0.004548417\n",
      "\t\t 9  --  0.006179247\n",
      "\t\t 10  --  0.008785875\n",
      "\t\t 11  --  0.02230807\n",
      "\t\t 12  --  0.10363737\n",
      "\t\t 13  --  0.10363737\n",
      "\t\t 14  --  0.10363737\n",
      "\t\t 15  --  0.10363737\n",
      "\t\t 16  --  0.10363737\n",
      "\t\t 17  --  0.2133652\n",
      "\t\t 18  --  0.3666688\n",
      "\t\t 19  --  0.499423\n",
      "\t\t 20  --  0.8521438\n",
      "18 - 25 changed.\n",
      "--------------------------\n",
      "******  5  ********\n",
      "\t\t 0  --  0.11300659\n",
      "\t\t 1  --  0.52507114\n",
      "27 - 3 changed.\n",
      "--------------------------\n",
      "******  6  ********\n",
      "\t\t 0  --  0.00042143624\n",
      "\t\t 1  --  0.00090534176\n",
      "\t\t 2  --  0.0012326337\n",
      "\t\t 3  --  0.0022738716\n",
      "\t\t 4  --  0.002621865\n",
      "\t\t 5  --  0.004095516\n",
      "\t\t 6  --  0.006105184\n",
      "\t\t 7  --  0.010004347\n",
      "\t\t 8  --  0.015898094\n",
      "\t\t 9  --  0.015898094\n",
      "\t\t 10  --  0.026303587\n",
      "\t\t 11  --  0.0530957\n",
      "\t\t 12  --  0.0822619\n",
      "\t\t 13  --  0.0822619\n",
      "\t\t 14  --  0.0822619\n",
      "\t\t 15  --  0.14044827\n",
      "\t\t 16  --  0.14044827\n",
      "\t\t 17  --  0.14662626\n",
      "\t\t 18  --  0.33448207\n",
      "\t\t 19  --  0.37523833\n",
      "\t\t 20  --  0.63711107\n",
      "40 - 24 changed.\n",
      "--------------------------\n",
      "******  7  ********\n",
      "\t\t 0  --  0.10721625\n",
      "\t\t 1  --  0.18760112\n",
      "\t\t 2  --  0.78860885\n",
      "57 - 3 changed.\n",
      "--------------------------\n",
      "******  8  ********\n",
      "\t\t 0  --  0.8323692\n",
      "58 - 1 changed.\n",
      "--------------------------\n",
      "******  9  ********\n",
      "\t\t 0  --  0.36571142\n",
      "\t\t 1  --  0.56981206\n",
      "82 - 3 changed.\n",
      "--------------------------\n",
      "******  10  ********\n",
      "\t\t 0  --  0.00024738375\n",
      "\t\t 1  --  0.0005180318\n",
      "\t\t 2  --  0.017406644\n",
      "\t\t 3  --  0.050103683\n",
      "\t\t 4  --  0.16957799\n",
      "\t\t 5  --  0.3216442\n",
      "\t\t 6  --  0.59389323\n",
      "86 - 9 changed.\n",
      "--------------------------\n",
      "******  11  ********\n",
      "\t\t 0  --  0.00075555535\n",
      "\t\t 1  --  0.0018917595\n",
      "\t\t 2  --  0.02146294\n",
      "\t\t 3  --  0.13271539\n",
      "\t\t 4  --  0.8240593\n",
      "116 - 5 changed.\n",
      "--------------------------\n",
      "******  12  ********\n",
      "\t\t 0  --  0.023768125\n",
      "\t\t 1  --  0.07246229\n",
      "\t\t 2  --  0.32825148\n",
      "\t\t 3  --  0.63903826\n",
      "127 - 5 changed.\n",
      "--------------------------\n",
      "******  13  ********\n",
      "\t\t 0  --  0.385406\n",
      "\t\t 1  --  0.8829553\n",
      "133 - 2 changed.\n",
      "--------------------------\n",
      "******  14  ********\n",
      "\t\t 0  --  0.042233758\n",
      "\t\t 1  --  0.1292183\n",
      "\t\t 2  --  0.26818925\n",
      "\t\t 3  --  0.5749079\n",
      "145 - 6 changed.\n",
      "--------------------------\n",
      "******  15  ********\n",
      "\t\t 0  --  0.78657943\n",
      "147 - 1 changed.\n",
      "--------------------------\n",
      "******  16  ********\n",
      "\t\t 0  --  0.93773377\n",
      "150 - 1 changed.\n",
      "--------------------------\n",
      "******  17  ********\n",
      "\t\t 0  --  0.0008528372\n",
      "\t\t 1  --  0.0019917628\n",
      "\t\t 2  --  0.0041931723\n",
      "\t\t 3  --  0.0057497174\n",
      "\t\t 4  --  0.009122723\n",
      "\t\t 5  --  0.01265144\n",
      "\t\t 6  --  0.019081617\n",
      "\t\t 7  --  0.024992099\n",
      "\t\t 8  --  0.030091478\n",
      "\t\t 9  --  0.03751634\n",
      "\t\t 10  --  0.18660231\n",
      "\t\t 11  --  0.18660231\n",
      "\t\t 12  --  0.18660231\n",
      "\t\t 13  --  0.20758407\n",
      "\t\t 14  --  0.3048751\n",
      "\t\t 15  --  0.3048751\n",
      "\t\t 16  --  0.33319232\n",
      "\t\t 17  --  0.41071898\n",
      "\t\t 18  --  0.46164232\n",
      "\t\t 19  --  0.46164232\n",
      "\t\t 20  --  0.4732525\n",
      "\t\t 21  --  0.49321935\n",
      "\t\t 22  --  0.55373996\n",
      "160 - 22 changed.\n",
      "--------------------------\n",
      "******  18  ********\n",
      "\t\t 0  --  0.15728432\n",
      "\t\t 1  --  0.21687846\n",
      "\t\t 2  --  0.5131407\n",
      "193 - 4 changed.\n",
      "--------------------------\n",
      "******  19  ********\n",
      "\t\t 0  --  0.13037115\n",
      "\t\t 1  --  0.15513974\n",
      "\t\t 2  --  0.18238884\n",
      "\t\t 3  --  0.18846603\n",
      "\t\t 4  --  0.18846603\n",
      "\t\t 5  --  0.20240976\n",
      "\t\t 6  --  0.22155102\n",
      "\t\t 7  --  0.23680769\n",
      "\t\t 8  --  0.8013689\n",
      "201 - 5 changed.\n",
      "--------------------------\n",
      "******  20  ********\n",
      "\t\t 0  --  0.06649446\n",
      "\t\t 1  --  0.088599876\n",
      "\t\t 2  --  0.21849939\n",
      "\t\t 3  --  0.21849939\n",
      "\t\t 4  --  0.38439748\n",
      "\t\t 5  --  0.4414507\n",
      "\t\t 6  --  0.4414507\n",
      "\t\t 7  --  0.5677795\n",
      "220 - 10 changed.\n",
      "--------------------------\n",
      "******  21  ********\n",
      "\t\t 0  --  0.005302601\n",
      "\t\t 1  --  0.035521585\n",
      "\t\t 2  --  0.105770096\n",
      "\t\t 3  --  0.105770096\n",
      "\t\t 4  --  0.14046225\n",
      "\t\t 5  --  0.28392804\n",
      "\t\t 6  --  0.77300906\n",
      "223 - 9 changed.\n",
      "--------------------------\n",
      "******  22  ********\n",
      "\t\t 0  --  0.36625713\n",
      "\t\t 1  --  0.6752362\n",
      "238 - 2 changed.\n",
      "--------------------------\n",
      "******  23  ********\n",
      "\t\t 0  --  0.086752795\n",
      "\t\t 1  --  0.23170385\n",
      "\t\t 2  --  0.5240704\n",
      "246 - 3 changed.\n",
      "--------------------------\n",
      "******  24  ********\n",
      "\t\t 0  --  0.0030346743\n",
      "\t\t 1  --  0.007583935\n",
      "\t\t 2  --  0.12055965\n",
      "\t\t 3  --  0.12055965\n",
      "\t\t 4  --  0.31829515\n",
      "\t\t 5  --  0.88164127\n",
      "248 - 5 changed.\n",
      "--------------------------\n",
      "******  25  ********\n",
      "\t\t 0  --  0.015823416\n",
      "\t\t 1  --  0.06346188\n",
      "\t\t 2  --  0.10399583\n",
      "\t\t 3  --  0.19670464\n",
      "\t\t 4  --  0.26415882\n",
      "\t\t 5  --  0.7816505\n",
      "250 - 7 changed.\n",
      "--------------------------\n",
      "******  26  ********\n",
      "\t\t 0  --  0.006412623\n",
      "\t\t 1  --  0.02141737\n",
      "\t\t 2  --  0.28666884\n",
      "\t\t 3  --  0.81660014\n",
      "255 - 7 changed.\n",
      "--------------------------\n",
      "******  27  ********\n",
      "\t\t 0  --  0.025488785\n",
      "\t\t 1  --  0.026948927\n",
      "\t\t 2  --  0.07126553\n",
      "\t\t 3  --  0.118706666\n",
      "\t\t 4  --  0.2588122\n",
      "\t\t 5  --  0.33740208\n",
      "\t\t 6  --  0.33740208\n",
      "\t\t 7  --  0.8093377\n",
      "256 - 9 changed.\n",
      "--------------------------\n",
      "******  28  ********\n",
      "\t\t 0  --  0.008904648\n",
      "\t\t 1  --  0.017661482\n",
      "\t\t 2  --  0.024770316\n",
      "\t\t 3  --  0.03965185\n",
      "\t\t 4  --  0.08999466\n",
      "\t\t 5  --  0.08999466\n",
      "\t\t 6  --  0.19026242\n",
      "\t\t 7  --  0.24199328\n",
      "\t\t 8  --  0.48855007\n",
      "\t\t 9  --  0.82699573\n",
      "280 - 12 changed.\n",
      "--------------------------\n",
      "******  29  ********\n",
      "\t\t 0  --  0.6211408\n",
      "299 - 1 changed.\n",
      "--------------------------\n",
      "******  30  ********\n",
      "\t\t 0  --  0.04969256\n",
      "\t\t 1  --  0.17948945\n",
      "\t\t 2  --  0.5340518\n",
      "316 - 5 changed.\n",
      "--------------------------\n",
      "******  31  ********\n",
      "\t\t 0  --  0.8191314\n",
      "321 - 1 changed.\n",
      "--------------------------\n",
      "******  32  ********\n",
      "\t\t 0  --  0.106331326\n",
      "\t\t 1  --  0.15714975\n",
      "\t\t 2  --  0.18860322\n",
      "\t\t 3  --  0.2847934\n",
      "\t\t 4  --  0.33369744\n",
      "\t\t 5  --  0.38472077\n",
      "\t\t 6  --  0.6743994\n",
      "330 - 10 changed.\n",
      "--------------------------\n",
      "******  33  ********\n",
      "\t\t 0  --  0.17624305\n",
      "\t\t 1  --  0.50528044\n",
      "339 - 3 changed.\n",
      "--------------------------\n",
      "******  34  ********\n",
      "\t\t 0  --  0.6568163\n",
      "341 - 1 changed.\n",
      "--------------------------\n",
      "******  35  ********\n",
      "\t\t 0  --  0.48004025\n",
      "\t\t 1  --  0.9007938\n",
      "353 - 3 changed.\n",
      "--------------------------\n",
      "******  36  ********\n",
      "\t\t 0  --  0.00092714373\n",
      "\t\t 1  --  0.10226388\n",
      "\t\t 2  --  0.1282964\n",
      "\t\t 3  --  0.1282964\n",
      "\t\t 4  --  0.28453633\n",
      "\t\t 5  --  0.29612625\n",
      "\t\t 6  --  0.4883983\n",
      "\t\t 7  --  0.54395765\n",
      "370 - 8 changed.\n",
      "--------------------------\n",
      "******  37  ********\n",
      "\t\t 0  --  0.007715799\n",
      "\t\t 1  --  0.20568399\n",
      "\t\t 2  --  0.49569938\n",
      "\t\t 3  --  0.6965257\n",
      "376 - 4 changed.\n",
      "--------------------------\n",
      "******  38  ********\n",
      "\t\t 0  --  0.37137365\n",
      "\t\t 1  --  0.4341948\n",
      "\t\t 2  --  0.79167706\n",
      "405 - 4 changed.\n",
      "--------------------------\n",
      "******  39  ********\n",
      "\t\t 0  --  0.001143047\n",
      "\t\t 1  --  0.006102639\n",
      "\t\t 2  --  0.011176887\n",
      "\t\t 3  --  0.5074052\n",
      "423 - 3 changed.\n",
      "--------------------------\n",
      "******  40  ********\n",
      "\t\t 0  --  0.13021256\n",
      "\t\t 1  --  0.16525787\n",
      "\t\t 2  --  0.43040317\n",
      "\t\t 3  --  0.6148088\n",
      "455 - 7 changed.\n",
      "--------------------------\n",
      "******  41  ********\n",
      "\t\t 0  --  0.031873826\n",
      "\t\t 1  --  0.12580451\n",
      "\t\t 2  --  0.32949823\n",
      "\t\t 3  --  0.6530041\n",
      "483 - 5 changed.\n",
      "--------------------------\n",
      "******  42  ********\n",
      "\t\t 0  --  0.05551122\n",
      "\t\t 1  --  0.31248453\n",
      "\t\t 2  --  0.4121505\n",
      "\t\t 3  --  0.8519209\n",
      "486 - 6 changed.\n",
      "--------------------------\n",
      "******  43  ********\n",
      "\t\t 0  --  0.018109553\n",
      "\t\t 1  --  0.093076214\n",
      "\t\t 2  --  0.32042533\n",
      "\t\t 3  --  0.52569264\n",
      "496 - 7 changed.\n",
      "--------------------------\n",
      "******  44  ********\n",
      "\t\t 0  --  0.022223959\n",
      "\t\t 1  --  0.25385284\n",
      "\t\t 2  --  0.3242449\n",
      "\t\t 3  --  0.5114369\n",
      "502 - 5 changed.\n",
      "--------------------------\n",
      "******  45  ********\n",
      "\t\t 0  --  0.66844875\n",
      "503 - 1 changed.\n",
      "--------------------------\n",
      "******  46  ********\n",
      "\t\t 0  --  0.15565655\n",
      "\t\t 1  --  0.18954825\n",
      "\t\t 2  --  0.26337337\n",
      "\t\t 3  --  0.2928906\n",
      "\t\t 4  --  0.45650357\n",
      "\t\t 5  --  0.5536174\n",
      "511 - 10 changed.\n",
      "--------------------------\n",
      "******  47  ********\n",
      "\t\t 0  --  0.04673872\n",
      "\t\t 1  --  0.54533947\n",
      "516 - 3 changed.\n",
      "--------------------------\n",
      "******  48  ********\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\t\t 0  --  0.017261371\n",
      "\t\t 1  --  0.02037716\n",
      "\t\t 2  --  0.023573171\n",
      "\t\t 3  --  0.08914166\n",
      "\t\t 4  --  0.53655946\n",
      "517 - 6 changed.\n",
      "--------------------------\n",
      "******  49  ********\n",
      "\t\t 0  --  0.8403348\n",
      "529 - 1 changed.\n",
      "--------------------------\n",
      "******  50  ********\n",
      "\t\t 0  --  0.029262105\n",
      "\t\t 1  --  0.24247132\n",
      "\t\t 2  --  0.24247132\n",
      "\t\t 3  --  0.42393875\n",
      "\t\t 4  --  0.60402477\n",
      "531 - 6 changed.\n",
      "--------------------------\n",
      "******  51  ********\n",
      "\t\t 0  --  0.5046299\n",
      "533 - 1 changed.\n",
      "--------------------------\n",
      "******  52  ********\n",
      "\t\t 0  --  0.42939952\n",
      "\t\t 1  --  0.939006\n",
      "540 - 3 changed.\n",
      "--------------------------\n",
      "******  53  ********\n",
      "\t\t 0  --  0.6096896\n",
      "541 - 1 changed.\n",
      "--------------------------\n",
      "******  54  ********\n",
      "\t\t 0  --  0.4184872\n",
      "\t\t 1  --  0.5380146\n",
      "545 - 3 changed.\n",
      "--------------------------\n",
      "******  55  ********\n",
      "\t\t 0  --  0.00039928843\n",
      "\t\t 1  --  0.0005260108\n",
      "\t\t 2  --  0.002020025\n",
      "\t\t 3  --  0.002020025\n",
      "\t\t 4  --  0.0048475764\n",
      "\t\t 5  --  0.0066693197\n",
      "\t\t 6  --  0.013560566\n",
      "\t\t 7  --  0.015769396\n",
      "\t\t 8  --  0.030737424\n",
      "\t\t 9  --  0.070622034\n",
      "\t\t 10  --  0.070622034\n",
      "\t\t 11  --  0.07627547\n",
      "\t\t 12  --  0.08564012\n",
      "\t\t 13  --  0.21888176\n",
      "\t\t 14  --  0.26122186\n",
      "\t\t 15  --  0.4122939\n",
      "\t\t 16  --  0.4586995\n",
      "\t\t 17  --  0.72026\n",
      "547 - 20 changed.\n",
      "--------------------------\n",
      "******  56  ********\n",
      "\t\t 0  --  0.08598491\n",
      "\t\t 1  --  0.14794429\n",
      "\t\t 2  --  0.31591216\n",
      "\t\t 3  --  0.60351694\n",
      "553 - 6 changed.\n",
      "--------------------------\n",
      "******  57  ********\n",
      "\t\t 0  --  0.65705776\n",
      "585 - 1 changed.\n",
      "--------------------------\n",
      "******  58  ********\n",
      "\t\t 0  --  0.14409024\n",
      "\t\t 1  --  0.28516945\n",
      "\t\t 2  --  0.4300497\n",
      "\t\t 3  --  0.86963296\n",
      "596 - 7 changed.\n",
      "--------------------------\n",
      "******  59  ********\n",
      "\t\t 0  --  0.012837459\n",
      "\t\t 1  --  0.017848704\n",
      "\t\t 2  --  0.08382591\n",
      "\t\t 3  --  0.10542272\n",
      "\t\t 4  --  0.20477593\n",
      "\t\t 5  --  0.5454632\n",
      "601 - 8 changed.\n",
      "--------------------------\n",
      "******  60  ********\n",
      "\t\t 0  --  0.00394916\n",
      "\t\t 1  --  0.007933534\n",
      "\t\t 2  --  0.011893501\n",
      "\t\t 3  --  0.0170788\n",
      "\t\t 4  --  0.023267454\n",
      "\t\t 5  --  0.03531112\n",
      "\t\t 6  --  0.048828106\n",
      "\t\t 7  --  0.7767036\n",
      "609 - 10 changed.\n",
      "--------------------------\n",
      "******  61  ********\n",
      "\t\t 0  --  0.08326685\n",
      "\t\t 1  --  0.14671592\n",
      "\t\t 2  --  0.554514\n",
      "617 - 4 changed.\n",
      "--------------------------\n",
      "******  62  ********\n",
      "\t\t 0  --  0.9472961\n",
      "633 - 1 changed.\n",
      "--------------------------\n",
      "******  63  ********\n",
      "\t\t 0  --  3.5728244e-06\n",
      "\t\t 1  --  1.3427091e-05\n",
      "\t\t 2  --  8.575016e-05\n",
      "\t\t 3  --  0.00017661475\n",
      "\t\t 4  --  0.0006521091\n",
      "\t\t 5  --  0.0010795966\n",
      "\t\t 6  --  0.0015682231\n",
      "\t\t 7  --  0.0015682231\n",
      "\t\t 8  --  0.0033249406\n",
      "\t\t 9  --  0.007624951\n",
      "\t\t 10  --  0.012484126\n",
      "\t\t 11  --  0.022446772\n",
      "\t\t 12  --  0.040764555\n",
      "\t\t 13  --  0.04458263\n",
      "\t\t 14  --  0.04458263\n",
      "\t\t 15  --  0.052024983\n",
      "\t\t 16  --  0.06925163\n",
      "\t\t 17  --  0.07971761\n",
      "\t\t 18  --  0.83269536\n",
      "639 - 21 changed.\n",
      "--------------------------\n",
      "******  64  ********\n",
      "\t\t 0  --  0.65292823\n",
      "644 - 1 changed.\n",
      "--------------------------\n",
      "******  65  ********\n",
      "\t\t 0  --  0.76983446\n",
      "645 - 1 changed.\n",
      "--------------------------\n",
      "******  66  ********\n",
      "\t\t 0  --  0.00011332651\n",
      "\t\t 1  --  0.00033405272\n",
      "\t\t 2  --  0.0037379994\n",
      "\t\t 3  --  0.05004528\n",
      "\t\t 4  --  0.05004528\n",
      "\t\t 5  --  0.05004528\n",
      "\t\t 6  --  0.15498333\n",
      "\t\t 7  --  0.15498333\n",
      "\t\t 8  --  0.44036976\n",
      "\t\t 9  --  0.48944482\n",
      "\t\t 10  --  0.855917\n",
      "667 - 13 changed.\n",
      "--------------------------\n",
      "******  67  ********\n",
      "\t\t 0  --  0.008109161\n",
      "\t\t 1  --  0.010910141\n",
      "\t\t 2  --  0.02324422\n",
      "\t\t 3  --  0.10528547\n",
      "\t\t 4  --  0.10528547\n",
      "\t\t 5  --  0.10528547\n",
      "\t\t 6  --  0.15390992\n",
      "\t\t 7  --  0.46732438\n",
      "\t\t 8  --  0.7566981\n",
      "678 - 14 changed.\n",
      "--------------------------\n",
      "******  68  ********\n",
      "\t\t 0  --  0.026560837\n",
      "\t\t 1  --  0.04373494\n",
      "\t\t 2  --  0.16016312\n",
      "\t\t 3  --  0.6258028\n",
      "687 - 5 changed.\n",
      "--------------------------\n",
      "******  69  ********\n",
      "\t\t 0  --  0.10580626\n",
      "\t\t 1  --  0.41402897\n",
      "\t\t 2  --  0.7163644\n",
      "697 - 5 changed.\n",
      "--------------------------\n",
      "******  70  ********\n",
      "\t\t 0  --  0.031615183\n",
      "\t\t 1  --  0.108168595\n",
      "\t\t 2  --  0.19809514\n",
      "\t\t 3  --  0.380057\n",
      "\t\t 4  --  0.51057136\n",
      "737 - 6 changed.\n",
      "--------------------------\n",
      "******  71  ********\n",
      "\t\t 0  --  0.00032941013\n",
      "\t\t 1  --  0.013125847\n",
      "\t\t 2  --  0.02220567\n",
      "\t\t 3  --  0.02220567\n",
      "\t\t 4  --  0.41340467\n",
      "\t\t 5  --  0.5341814\n",
      "740 - 9 changed.\n",
      "--------------------------\n",
      "******  72  ********\n",
      "\t\t 0  --  0.5813546\n",
      "756 - 1 changed.\n",
      "--------------------------\n",
      "******  73  ********\n",
      "\t\t 0  --  0.002249971\n",
      "\t\t 1  --  0.010727711\n",
      "\t\t 2  --  0.10425584\n",
      "\t\t 3  --  0.10425584\n",
      "\t\t 4  --  0.7489373\n",
      "812 - 9 changed.\n",
      "--------------------------\n",
      "******  74  ********\n",
      "\t\t 0  --  0.3459879\n",
      "\t\t 1  --  0.47925642\n",
      "\t\t 2  --  0.9748274\n",
      "817 - 5 changed.\n",
      "--------------------------\n",
      "******  75  ********\n",
      "\t\t 0  --  0.08282522\n",
      "\t\t 1  --  0.29048672\n",
      "\t\t 2  --  0.6404124\n",
      "837 - 5 changed.\n",
      "--------------------------\n",
      "******  76  ********\n",
      "\t\t 0  --  0.001127523\n",
      "\t\t 1  --  0.0017372452\n",
      "\t\t 2  --  0.0043248856\n",
      "\t\t 3  --  0.0047929534\n",
      "\t\t 4  --  0.0060801725\n",
      "\t\t 5  --  0.007792612\n",
      "\t\t 6  --  0.007792612\n",
      "\t\t 7  --  0.00801194\n",
      "\t\t 8  --  0.013294976\n",
      "\t\t 9  --  0.015558276\n",
      "\t\t 10  --  0.019684767\n",
      "\t\t 11  --  0.019684767\n",
      "\t\t 12  --  0.03626266\n",
      "\t\t 13  --  0.068435825\n",
      "\t\t 14  --  0.10513817\n",
      "\t\t 15  --  0.10513817\n",
      "\t\t 16  --  0.10513817\n",
      "\t\t 17  --  0.11547238\n",
      "\t\t 18  --  0.2415046\n",
      "\t\t 19  --  0.27610624\n",
      "\t\t 20  --  0.34895083\n",
      "\t\t 21  --  0.5306578\n",
      "876 - 20 changed.\n",
      "--------------------------\n",
      "******  77  ********\n",
      "\t\t 0  --  0.95852673\n",
      "880 - 1 changed.\n",
      "--------------------------\n",
      "******  78  ********\n",
      "\t\t 0  --  0.25379625\n",
      "\t\t 1  --  0.5642128\n",
      "899 - 2 changed.\n",
      "--------------------------\n",
      "******  79  ********\n",
      "\t\t 0  --  0.019423444\n",
      "\t\t 1  --  0.12368821\n",
      "\t\t 2  --  0.23235834\n",
      "\t\t 3  --  0.31838876\n",
      "\t\t 4  --  0.51992893\n",
      "915 - 8 changed.\n",
      "--------------------------\n",
      "******  80  ********\n",
      "\t\t 0  --  0.22811466\n",
      "\t\t 1  --  0.29660213\n",
      "\t\t 2  --  0.55420566\n",
      "923 - 4 changed.\n",
      "--------------------------\n",
      "******  81  ********\n",
      "\t\t 0  --  0.033925097\n",
      "\t\t 1  --  0.06769932\n",
      "\t\t 2  --  0.17368586\n",
      "\t\t 3  --  0.17368586\n",
      "\t\t 4  --  0.26798505\n",
      "\t\t 5  --  0.28173432\n",
      "\t\t 6  --  0.28545803\n",
      "\t\t 7  --  0.49856937\n",
      "\t\t 8  --  0.5365944\n",
      "926 - 12 changed.\n",
      "--------------------------\n",
      "******  82  ********\n",
      "\t\t 0  --  0.76047397\n",
      "930 - 1 changed.\n",
      "--------------------------\n",
      "******  83  ********\n",
      "\t\t 0  --  0.09287606\n",
      "\t\t 1  --  0.30768925\n",
      "\t\t 2  --  0.96755576\n",
      "938 - 5 changed.\n",
      "--------------------------\n",
      "******  84  ********\n",
      "\t\t 0  --  0.79398435\n",
      "943 - 1 changed.\n",
      "--------------------------\n",
      "******  85  ********\n",
      "\t\t 0  --  0.0064599663\n",
      "\t\t 1  --  0.016082196\n",
      "\t\t 2  --  0.14553224\n",
      "\t\t 3  --  0.47372252\n",
      "\t\t 4  --  0.58641624\n",
      "948 - 7 changed.\n",
      "--------------------------\n",
      "******  86  ********\n",
      "\t\t 0  --  0.1088337\n",
      "\t\t 1  --  0.12722304\n",
      "\t\t 2  --  0.28780147\n",
      "\t\t 3  --  0.3477829\n",
      "\t\t 4  --  0.40922108\n",
      "\t\t 5  --  0.40922108\n",
      "\t\t 6  --  0.494971\n",
      "\t\t 7  --  0.6194315\n",
      "957 - 10 changed.\n",
      "--------------------------\n",
      "******  87  ********\n",
      "\t\t 0  --  0.12162275\n",
      "\t\t 1  --  0.1748016\n",
      "\t\t 2  --  0.30249283\n",
      "\t\t 3  --  0.5435069\n",
      "959 - 8 changed.\n",
      "--------------------------\n",
      "******  88  ********\n",
      "\t\t 0  --  1.01079295e-05\n",
      "\t\t 1  --  1.2328613e-05\n",
      "\t\t 2  --  2.05508e-05\n",
      "\t\t 3  --  3.3267388e-05\n",
      "\t\t 4  --  3.3267388e-05\n",
      "\t\t 5  --  0.000115238676\n",
      "\t\t 6  --  0.00055360957\n",
      "\t\t 7  --  0.00063548336\n",
      "\t\t 8  --  0.003056517\n",
      "\t\t 9  --  0.003056517\n",
      "\t\t 10  --  0.003056517\n",
      "\t\t 11  --  0.003056517\n",
      "\t\t 12  --  0.003056517\n",
      "\t\t 13  --  0.0105654625\n",
      "\t\t 14  --  0.018983644\n",
      "\t\t 15  --  0.018983644\n",
      "\t\t 16  --  0.052105933\n",
      "\t\t 17  --  0.052105933\n",
      "\t\t 18  --  0.052105933\n",
      "\t\t 19  --  0.052105933\n",
      "\t\t 20  --  0.10868549\n",
      "\t\t 21  --  0.10868549\n",
      "\t\t 22  --  0.30566013\n",
      "\t\t 23  --  0.30566013\n",
      "\t\t 24  --  0.43640092\n",
      "\t\t 25  --  0.43640092\n",
      "\t\t 26  --  0.51830906\n",
      "970 - 28 changed.\n",
      "--------------------------\n",
      "******  89  ********\n",
      "\t\t 0  --  0.0038334557\n",
      "\t\t 1  --  0.008022855\n",
      "\t\t 2  --  0.020750385\n",
      "\t\t 3  --  0.021678368\n",
      "\t\t 4  --  0.055227417\n",
      "\t\t 5  --  0.11436785\n",
      "\t\t 6  --  0.12159301\n",
      "\t\t 7  --  0.13028477\n",
      "\t\t 8  --  0.16040783\n",
      "\t\t 9  --  0.3759118\n",
      "\t\t 10  --  0.6037535\n",
      "1007 - 12 changed.\n",
      "--------------------------\n",
      "******  90  ********\n",
      "\t\t 0  --  0.006767632\n",
      "\t\t 1  --  0.0168025\n",
      "\t\t 2  --  0.01902501\n",
      "\t\t 3  --  0.03600788\n",
      "\t\t 4  --  0.07857993\n",
      "\t\t 5  --  0.11074796\n",
      "\t\t 6  --  0.41046256\n",
      "\t\t 7  --  0.6544896\n",
      "1008 - 11 changed.\n",
      "--------------------------\n",
      "******  91  ********\n",
      "\t\t 0  --  0.8799318\n",
      "1010 - 1 changed.\n",
      "--------------------------\n",
      "******  92  ********\n",
      "\t\t 0  --  0.1081787\n",
      "\t\t 1  --  0.4278887\n",
      "\t\t 2  --  0.6338213\n",
      "1020 - 4 changed.\n",
      "--------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "******  93  ********\n",
      "\t\t 0  --  0.74539155\n",
      "1040 - 1 changed.\n",
      "--------------------------\n",
      "******  94  ********\n",
      "\t\t 0  --  0.0011191729\n",
      "\t\t 1  --  0.012057994\n",
      "\t\t 2  --  0.08946816\n",
      "\t\t 3  --  0.45718634\n",
      "\t\t 4  --  0.8232168\n",
      "1045 - 5 changed.\n",
      "--------------------------\n",
      "******  95  ********\n",
      "\t\t 0  --  0.023614038\n",
      "\t\t 1  --  0.40257096\n",
      "\t\t 2  --  0.49651057\n",
      "\t\t 3  --  0.98317194\n",
      "1057 - 5 changed.\n",
      "--------------------------\n",
      "******  96  ********\n",
      "\t\t 0  --  0.023407836\n",
      "\t\t 1  --  0.57342935\n",
      "1066 - 1 changed.\n",
      "--------------------------\n",
      "******  97  ********\n",
      "\t\t 0  --  4.483463e-06\n",
      "\t\t 1  --  5.2084292e-06\n",
      "\t\t 2  --  1.7096958e-05\n",
      "\t\t 3  --  1.7096958e-05\n",
      "\t\t 4  --  2.7275075e-05\n",
      "\t\t 5  --  0.00013349178\n",
      "\t\t 6  --  0.00013349178\n",
      "\t\t 7  --  0.00013349178\n",
      "\t\t 8  --  0.016644001\n",
      "\t\t 9  --  0.03548153\n",
      "\t\t 10  --  0.03548153\n",
      "\t\t 11  --  0.03548153\n",
      "\t\t 12  --  0.13038312\n",
      "\t\t 13  --  0.13038312\n",
      "\t\t 14  --  0.13038312\n",
      "\t\t 15  --  0.13038312\n",
      "\t\t 16  --  0.13038312\n",
      "\t\t 17  --  0.29166052\n",
      "\t\t 18  --  0.30049193\n",
      "\t\t 19  --  0.88937527\n",
      "1085 - 22 changed.\n",
      "--------------------------\n",
      "******  98  ********\n",
      "\t\t 0  --  0.21590598\n",
      "\t\t 1  --  0.37309197\n",
      "\t\t 2  --  0.90253246\n",
      "1089 - 5 changed.\n",
      "--------------------------\n",
      "******  99  ********\n",
      "\t\t 0  --  0.023443738\n",
      "\t\t 1  --  0.121703014\n",
      "\t\t 2  --  0.121703014\n",
      "\t\t 3  --  0.12344689\n",
      "\t\t 4  --  0.5034571\n",
      "1096 - 7 changed.\n",
      "--------------------------\n",
      "******  100  ********\n",
      "\t\t 0  --  0.43178952\n",
      "\t\t 1  --  0.8388736\n",
      "1098 - 3 changed.\n",
      "--------------------------\n",
      "******  101  ********\n",
      "\t\t 0  --  0.021202013\n",
      "\t\t 1  --  0.045220573\n",
      "\t\t 2  --  0.07388019\n",
      "\t\t 3  --  0.109324284\n",
      "\t\t 4  --  0.14568415\n",
      "\t\t 5  --  0.14938012\n",
      "\t\t 6  --  0.14938012\n",
      "\t\t 7  --  0.1759189\n",
      "\t\t 8  --  0.20046496\n",
      "\t\t 9  --  0.22422913\n",
      "\t\t 10  --  0.23113067\n",
      "\t\t 11  --  0.23113067\n",
      "\t\t 12  --  0.2398964\n",
      "\t\t 13  --  0.2483257\n",
      "\t\t 14  --  0.2483257\n",
      "\t\t 15  --  0.26471418\n",
      "\t\t 16  --  0.27358752\n",
      "\t\t 17  --  0.27651376\n",
      "\t\t 18  --  0.29030782\n",
      "\t\t 19  --  0.29030782\n",
      "\t\t 20  --  0.29030782\n",
      "\t\t 21  --  0.3020722\n",
      "\t\t 22  --  0.3020722\n",
      "\t\t 23  --  0.3020722\n",
      "\t\t 24  --  0.3020722\n",
      "\t\t 25  --  0.3020722\n",
      "\t\t 26  --  0.3020722\n",
      "\t\t 27  --  0.3020722\n",
      "\t\t 28  --  0.3020722\n",
      "\t\t 29  --  0.3020722\n",
      "1104 failed\n",
      "--------------------------\n",
      "******  102  ********\n",
      "\t\t 0  --  0.25806355\n",
      "\t\t 1  --  0.57494056\n",
      "1111 - 3 changed.\n",
      "--------------------------\n",
      "******  103  ********\n",
      "\t\t 0  --  0.2659441\n",
      "\t\t 1  --  0.53009474\n",
      "1112 - 2 changed.\n",
      "--------------------------\n",
      "******  104  ********\n",
      "\t\t 0  --  0.0067053395\n",
      "\t\t 1  --  0.019268593\n",
      "\t\t 2  --  0.04438226\n",
      "\t\t 3  --  0.05398338\n",
      "\t\t 4  --  0.27127892\n",
      "\t\t 5  --  0.39907417\n",
      "\t\t 6  --  0.5809342\n",
      "1116 - 10 changed.\n",
      "--------------------------\n",
      "******  105  ********\n",
      "\t\t 0  --  0.54862994\n",
      "1123 - 1 changed.\n",
      "--------------------------\n",
      "******  106  ********\n",
      "\t\t 0  --  0.013955146\n",
      "\t\t 1  --  0.04406336\n",
      "\t\t 2  --  0.13458386\n",
      "\t\t 3  --  0.13458386\n",
      "\t\t 4  --  0.5311019\n",
      "1124 - 3 changed.\n",
      "--------------------------\n",
      "******  107  ********\n",
      "\t\t 0  --  0.0072529376\n",
      "\t\t 1  --  0.029242935\n",
      "\t\t 2  --  0.029242935\n",
      "\t\t 3  --  0.04842554\n",
      "\t\t 4  --  0.059231542\n",
      "\t\t 5  --  0.070875965\n",
      "\t\t 6  --  0.17799018\n",
      "\t\t 7  --  0.17799018\n",
      "\t\t 8  --  0.17799018\n",
      "\t\t 9  --  0.17799018\n",
      "\t\t 10  --  0.23829396\n",
      "\t\t 11  --  0.4150678\n",
      "\t\t 12  --  0.47271994\n",
      "\t\t 13  --  0.47271994\n",
      "\t\t 14  --  0.47271994\n",
      "\t\t 15  --  0.48539144\n",
      "\t\t 16  --  0.57963127\n",
      "1126 - 17 changed.\n",
      "--------------------------\n",
      "******  108  ********\n",
      "\t\t 0  --  0.998638\n",
      "1141 - 1 changed.\n",
      "--------------------------\n",
      "******  109  ********\n",
      "\t\t 0  --  0.0025570614\n",
      "\t\t 1  --  0.017013595\n",
      "\t\t 2  --  0.029735219\n",
      "\t\t 3  --  0.06609572\n",
      "\t\t 4  --  0.33317235\n",
      "\t\t 5  --  0.33317235\n",
      "\t\t 6  --  0.6275844\n",
      "1159 - 10 changed.\n",
      "--------------------------\n",
      "******  110  ********\n",
      "\t\t 0  --  0.01689323\n",
      "\t\t 1  --  0.01689323\n",
      "\t\t 2  --  0.026556391\n",
      "\t\t 3  --  0.041801814\n",
      "\t\t 4  --  0.16337892\n",
      "\t\t 5  --  0.5064045\n",
      "1162 - 9 changed.\n",
      "--------------------------\n",
      "******  111  ********\n",
      "\t\t 0  --  0.0031366555\n",
      "\t\t 1  --  0.05249898\n",
      "\t\t 2  --  0.07102047\n",
      "\t\t 3  --  0.24084368\n",
      "\t\t 4  --  0.8493419\n",
      "1175 - 7 changed.\n",
      "--------------------------\n",
      "******  112  ********\n",
      "\t\t 0  --  0.57657206\n",
      "1177 - 1 changed.\n",
      "--------------------------\n",
      "******  113  ********\n",
      "\t\t 0  --  0.005864524\n",
      "\t\t 1  --  0.01211489\n",
      "\t\t 2  --  0.0395294\n",
      "\t\t 3  --  0.0887053\n",
      "\t\t 4  --  0.0887053\n",
      "\t\t 5  --  0.22669536\n",
      "\t\t 6  --  0.22669536\n",
      "\t\t 7  --  0.39808536\n",
      "\t\t 8  --  0.53510517\n",
      "1178 - 9 changed.\n",
      "--------------------------\n",
      "******  114  ********\n",
      "\t\t 0  --  0.20245011\n",
      "\t\t 1  --  0.37739056\n",
      "\t\t 2  --  0.5757929\n",
      "1189 - 6 changed.\n",
      "--------------------------\n",
      "******  115  ********\n",
      "\t\t 0  --  0.50105345\n",
      "1218 - 1 changed.\n",
      "--------------------------\n",
      "******  116  ********\n",
      "\t\t 0  --  0.003883774\n",
      "\t\t 1  --  0.0076000188\n",
      "\t\t 2  --  0.024894658\n",
      "\t\t 3  --  0.054089267\n",
      "\t\t 4  --  0.52365017\n",
      "1233 - 5 changed.\n",
      "--------------------------\n",
      "******  117  ********\n",
      "\t\t 0  --  0.10831747\n",
      "\t\t 1  --  0.7529184\n",
      "1246 - 3 changed.\n",
      "--------------------------\n",
      "******  118  ********\n",
      "\t\t 0  --  0.3731061\n",
      "\t\t 1  --  0.785141\n",
      "1258 - 3 changed.\n",
      "--------------------------\n",
      "******  119  ********\n",
      "\t\t 0  --  0.06478681\n",
      "\t\t 1  --  0.16026592\n",
      "\t\t 2  --  0.18772149\n",
      "\t\t 3  --  0.5462999\n",
      "1264 - 6 changed.\n",
      "--------------------------\n",
      "******  120  ********\n",
      "\t\t 0  --  0.005539718\n",
      "\t\t 1  --  0.028310152\n",
      "\t\t 2  --  0.028310152\n",
      "\t\t 3  --  0.08010083\n",
      "\t\t 4  --  0.08010083\n",
      "\t\t 5  --  0.17426209\n",
      "\t\t 6  --  0.18414275\n",
      "\t\t 7  --  0.3743389\n",
      "\t\t 8  --  0.5515954\n",
      "1276 - 9 changed.\n",
      "--------------------------\n",
      "******  121  ********\n",
      "\t\t 0  --  0.0041748392\n",
      "\t\t 1  --  0.004346945\n",
      "\t\t 2  --  0.8568102\n",
      "1278 - 2 changed.\n",
      "--------------------------\n",
      "******  122  ********\n",
      "\t\t 0  --  0.8470487\n",
      "1290 - 1 changed.\n",
      "--------------------------\n",
      "******  123  ********\n",
      "\t\t 0  --  0.39496177\n",
      "\t\t 1  --  0.83652514\n",
      "1295 - 1 changed.\n",
      "--------------------------\n",
      "******  124  ********\n",
      "\t\t 0  --  0.27762902\n",
      "\t\t 1  --  0.3945489\n",
      "\t\t 2  --  0.58919346\n",
      "1303 - 4 changed.\n",
      "--------------------------\n",
      "******  125  ********\n",
      "\t\t 0  --  0.18300982\n",
      "\t\t 1  --  0.2562215\n",
      "\t\t 2  --  0.4025476\n",
      "\t\t 3  --  0.49786797\n",
      "\t\t 4  --  0.623174\n",
      "1304 - 5 changed.\n",
      "--------------------------\n",
      "******  126  ********\n",
      "\t\t 0  --  0.013315822\n",
      "\t\t 1  --  0.052922815\n",
      "\t\t 2  --  0.23715594\n",
      "\t\t 3  --  0.23715594\n",
      "\t\t 4  --  0.7737002\n",
      "1311 - 4 changed.\n",
      "--------------------------\n",
      "******  127  ********\n",
      "\t\t 0  --  0.0010438991\n",
      "\t\t 1  --  0.002380002\n",
      "\t\t 2  --  0.0039985646\n",
      "\t\t 3  --  0.0073454366\n",
      "\t\t 4  --  0.014086701\n",
      "\t\t 5  --  0.20376858\n",
      "\t\t 6  --  0.685833\n",
      "1312 - 12 changed.\n",
      "--------------------------\n",
      "******  128  ********\n",
      "\t\t 0  --  0.08236279\n",
      "\t\t 1  --  0.12983492\n",
      "\t\t 2  --  0.1964272\n",
      "\t\t 3  --  0.60090894\n",
      "1322 - 4 changed.\n",
      "--------------------------\n",
      "******  129  ********\n",
      "\t\t 0  --  0.014007246\n",
      "\t\t 1  --  0.036998395\n",
      "\t\t 2  --  0.21789582\n",
      "\t\t 3  --  0.3680209\n",
      "\t\t 4  --  0.68714505\n",
      "1325 - 6 changed.\n",
      "--------------------------\n",
      "******  130  ********\n",
      "\t\t 0  --  0.0025966489\n",
      "\t\t 1  --  0.0026470793\n",
      "\t\t 2  --  0.010656381\n",
      "\t\t 3  --  0.09825556\n",
      "\t\t 4  --  0.09825556\n",
      "\t\t 5  --  0.17995757\n",
      "\t\t 6  --  0.17995757\n",
      "\t\t 7  --  0.5979984\n",
      "1328 - 8 changed.\n",
      "--------------------------\n",
      "******  131  ********\n",
      "\t\t 0  --  0.0046505295\n",
      "\t\t 1  --  0.164875\n",
      "\t\t 2  --  0.77022344\n",
      "1348 - 3 changed.\n",
      "--------------------------\n",
      "******  132  ********\n",
      "\t\t 0  --  0.7984208\n",
      "1353 - 1 changed.\n",
      "--------------------------\n",
      "******  133  ********\n",
      "\t\t 0  --  0.03699905\n",
      "\t\t 1  --  0.051122826\n",
      "\t\t 2  --  0.26740932\n",
      "\t\t 3  --  0.36465567\n",
      "\t\t 4  --  0.567571\n",
      "1358 - 6 changed.\n",
      "--------------------------\n",
      "******  134  ********\n",
      "\t\t 0  --  0.00561325\n",
      "\t\t 1  --  0.012685795\n",
      "\t\t 2  --  0.93875796\n",
      "1364 - 3 changed.\n",
      "--------------------------\n",
      "******  135  ********\n",
      "\t\t 0  --  3.5993926e-05\n",
      "\t\t 1  --  0.00014479678\n",
      "\t\t 2  --  0.00044351828\n",
      "\t\t 3  --  0.0009780788\n",
      "\t\t 4  --  0.0009780788\n",
      "\t\t 5  --  0.0038873928\n",
      "\t\t 6  --  0.010138161\n",
      "\t\t 7  --  0.017152239\n",
      "\t\t 8  --  0.050207164\n",
      "\t\t 9  --  0.050207164\n",
      "\t\t 10  --  0.050660383\n",
      "\t\t 11  --  0.76868886\n",
      "1377 - 14 changed.\n",
      "--------------------------\n",
      "******  136  ********\n",
      "\t\t 0  --  0.8590366\n",
      "1378 - 1 changed.\n",
      "--------------------------\n",
      "******  137  ********\n",
      "\t\t 0  --  0.0009812351\n",
      "\t\t 1  --  0.0035439394\n",
      "\t\t 2  --  0.0061345734\n",
      "\t\t 3  --  0.0071470644\n",
      "\t\t 4  --  0.012725955\n",
      "\t\t 5  --  0.034604467\n",
      "\t\t 6  --  0.05805044\n",
      "\t\t 7  --  0.05805044\n",
      "\t\t 8  --  0.05805044\n",
      "\t\t 9  --  0.08302801\n",
      "\t\t 10  --  0.08467074\n",
      "\t\t 11  --  0.13010639\n",
      "\t\t 12  --  0.13010639\n",
      "\t\t 13  --  0.13010639\n",
      "\t\t 14  --  0.15749611\n",
      "\t\t 15  --  0.26112506\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\t\t 16  --  0.27449238\n",
      "\t\t 17  --  0.46243197\n",
      "\t\t 18  --  0.5401243\n",
      "1394 - 24 changed.\n",
      "--------------------------\n",
      "******  138  ********\n",
      "\t\t 0  --  0.01837934\n",
      "\t\t 1  --  0.032186992\n",
      "\t\t 2  --  0.070028216\n",
      "\t\t 3  --  0.070028216\n",
      "\t\t 4  --  0.096304245\n",
      "\t\t 5  --  0.19163771\n",
      "\t\t 6  --  0.20079419\n",
      "\t\t 7  --  0.25623515\n",
      "\t\t 8  --  0.40685016\n",
      "\t\t 9  --  0.5189224\n",
      "1405 - 13 changed.\n",
      "--------------------------\n",
      "******  139  ********\n",
      "\t\t 0  --  0.0008616909\n",
      "\t\t 1  --  0.0025734382\n",
      "\t\t 2  --  0.094167694\n",
      "\t\t 3  --  0.21284099\n",
      "\t\t 4  --  0.43182877\n",
      "\t\t 5  --  0.7427157\n",
      "1423 - 8 changed.\n",
      "--------------------------\n",
      "******  140  ********\n",
      "\t\t 0  --  0.103433155\n",
      "\t\t 1  --  0.56482273\n",
      "1426 - 1 changed.\n",
      "--------------------------\n",
      "******  141  ********\n",
      "\t\t 0  --  0.012065027\n",
      "\t\t 1  --  0.15197961\n",
      "\t\t 2  --  0.8119772\n",
      "1434 - 5 changed.\n",
      "--------------------------\n",
      "******  142  ********\n",
      "\t\t 0  --  0.013773772\n",
      "\t\t 1  --  0.018129138\n",
      "\t\t 2  --  0.09367223\n",
      "\t\t 3  --  0.29025075\n",
      "\t\t 4  --  0.33429804\n",
      "\t\t 5  --  0.34540194\n",
      "\t\t 6  --  0.38546276\n",
      "\t\t 7  --  0.80289656\n",
      "1476 - 7 changed.\n",
      "--------------------------\n",
      "******  143  ********\n",
      "\t\t 0  --  0.017672936\n",
      "\t\t 1  --  0.043033835\n",
      "\t\t 2  --  0.05148815\n",
      "\t\t 3  --  0.23454364\n",
      "\t\t 4  --  0.6686413\n",
      "1484 - 3 changed.\n",
      "--------------------------\n",
      "******  144  ********\n",
      "\t\t 0  --  0.63778716\n",
      "1494 - 1 changed.\n",
      "--------------------------\n",
      "******  145  ********\n",
      "\t\t 0  --  0.9100242\n",
      "1502 - 1 changed.\n",
      "--------------------------\n",
      "******  146  ********\n",
      "\t\t 0  --  0.0116774915\n",
      "\t\t 1  --  0.025302636\n",
      "\t\t 2  --  0.026954977\n",
      "\t\t 3  --  0.062215496\n",
      "\t\t 4  --  0.07507557\n",
      "\t\t 5  --  0.21129446\n",
      "\t\t 6  --  0.29794386\n",
      "\t\t 7  --  0.41360366\n",
      "\t\t 8  --  0.66700566\n",
      "1506 - 12 changed.\n",
      "--------------------------\n",
      "******  147  ********\n",
      "\t\t 0  --  0.0022669605\n",
      "\t\t 1  --  0.005464979\n",
      "\t\t 2  --  0.0121803675\n",
      "\t\t 3  --  0.021522664\n",
      "\t\t 4  --  0.08440108\n",
      "\t\t 5  --  0.10854215\n",
      "\t\t 6  --  0.15660205\n",
      "\t\t 7  --  0.21050689\n",
      "\t\t 8  --  0.21241865\n",
      "\t\t 9  --  0.81797624\n",
      "1508 - 14 changed.\n",
      "--------------------------\n",
      "******  148  ********\n",
      "\t\t 0  --  0.1556473\n",
      "\t\t 1  --  0.18688758\n",
      "\t\t 2  --  0.4115148\n",
      "\t\t 3  --  0.85159004\n",
      "1512 - 6 changed.\n",
      "--------------------------\n",
      "******  149  ********\n",
      "\t\t 0  --  0.112553135\n",
      "\t\t 1  --  0.9758982\n",
      "1513 - 2 changed.\n",
      "--------------------------\n",
      "******  150  ********\n",
      "\t\t 0  --  0.0026317893\n",
      "\t\t 1  --  0.010437174\n",
      "\t\t 2  --  0.07409625\n",
      "\t\t 3  --  0.13740492\n",
      "\t\t 4  --  0.31853324\n",
      "\t\t 5  --  0.5621235\n",
      "1521 - 9 changed.\n",
      "--------------------------\n",
      "******  151  ********\n",
      "\t\t 0  --  0.010410524\n",
      "\t\t 1  --  0.22780676\n",
      "\t\t 2  --  0.22780676\n",
      "\t\t 3  --  0.598027\n",
      "1527 - 5 changed.\n",
      "--------------------------\n",
      "******  152  ********\n",
      "\t\t 0  --  0.29229367\n",
      "\t\t 1  --  0.47001126\n",
      "\t\t 2  --  0.9903948\n",
      "1553 - 6 changed.\n",
      "--------------------------\n",
      "******  153  ********\n",
      "\t\t 0  --  0.015886609\n",
      "\t\t 1  --  0.23223642\n",
      "\t\t 2  --  0.23223642\n",
      "\t\t 3  --  0.6287947\n",
      "1560 - 3 changed.\n",
      "--------------------------\n",
      "******  154  ********\n",
      "\t\t 0  --  0.60583067\n",
      "1576 - 1 changed.\n",
      "--------------------------\n",
      "******  155  ********\n",
      "\t\t 0  --  0.00044315506\n",
      "\t\t 1  --  0.00091748446\n",
      "\t\t 2  --  0.0016289769\n",
      "\t\t 3  --  0.00402676\n",
      "\t\t 4  --  0.0072662397\n",
      "\t\t 5  --  0.053759705\n",
      "\t\t 6  --  0.053759705\n",
      "\t\t 7  --  0.71469045\n",
      "1579 - 11 changed.\n",
      "--------------------------\n",
      "******  156  ********\n",
      "\t\t 0  --  0.786493\n",
      "1586 - 1 changed.\n",
      "--------------------------\n",
      "******  157  ********\n",
      "\t\t 0  --  0.6531738\n",
      "1590 - 1 changed.\n",
      "--------------------------\n",
      "******  158  ********\n",
      "\t\t 0  --  0.7279479\n",
      "1600 - 1 changed.\n",
      "--------------------------\n",
      "******  159  ********\n",
      "\t\t 0  --  0.0076391804\n",
      "\t\t 1  --  0.0076391804\n",
      "\t\t 2  --  0.12398313\n",
      "\t\t 3  --  0.25889662\n",
      "\t\t 4  --  0.7194838\n",
      "1606 - 7 changed.\n",
      "--------------------------\n",
      "******  160  ********\n",
      "\t\t 0  --  0.00079965946\n",
      "\t\t 1  --  0.02372893\n",
      "\t\t 2  --  0.045374744\n",
      "\t\t 3  --  0.21042266\n",
      "\t\t 4  --  0.3628048\n",
      "\t\t 5  --  0.5661166\n",
      "1607 - 6 changed.\n",
      "--------------------------\n",
      "******  161  ********\n",
      "\t\t 0  --  0.024951434\n",
      "\t\t 1  --  0.10631689\n",
      "\t\t 2  --  0.2073225\n",
      "\t\t 3  --  0.30534852\n",
      "\t\t 4  --  0.51656574\n",
      "1608 - 7 changed.\n",
      "--------------------------\n",
      "******  162  ********\n",
      "\t\t 0  --  0.17952709\n",
      "\t\t 1  --  0.7688521\n",
      "1620 - 2 changed.\n",
      "--------------------------\n",
      "******  163  ********\n",
      "\t\t 0  --  0.23726428\n",
      "\t\t 1  --  0.32102802\n",
      "\t\t 2  --  0.39576876\n",
      "\t\t 3  --  0.55942947\n",
      "1626 - 4 changed.\n",
      "--------------------------\n",
      "******  164  ********\n",
      "\t\t 0  --  0.34320855\n",
      "\t\t 1  --  0.6911023\n",
      "1629 - 3 changed.\n",
      "--------------------------\n",
      "******  165  ********\n",
      "\t\t 0  --  0.6538141\n",
      "1630 - 1 changed.\n",
      "--------------------------\n",
      "******  166  ********\n",
      "\t\t 0  --  0.008740186\n",
      "\t\t 1  --  0.010051881\n",
      "\t\t 2  --  0.011175861\n",
      "\t\t 3  --  0.017454159\n",
      "\t\t 4  --  0.029132226\n",
      "\t\t 5  --  0.038244475\n",
      "\t\t 6  --  0.099580355\n",
      "\t\t 7  --  0.099580355\n",
      "\t\t 8  --  0.099580355\n",
      "\t\t 9  --  0.20042872\n",
      "\t\t 10  --  0.27768576\n",
      "\t\t 11  --  0.27768576\n",
      "\t\t 12  --  0.38553768\n",
      "\t\t 13  --  0.63437766\n",
      "1644 - 13 changed.\n",
      "--------------------------\n",
      "******  167  ********\n",
      "\t\t 0  --  0.041187573\n",
      "\t\t 1  --  0.3091128\n",
      "\t\t 2  --  0.6013486\n",
      "1653 - 5 changed.\n",
      "--------------------------\n",
      "******  168  ********\n",
      "\t\t 0  --  0.00086346787\n",
      "\t\t 1  --  0.009636137\n",
      "\t\t 2  --  0.018052693\n",
      "\t\t 3  --  0.021305095\n",
      "\t\t 4  --  0.096891835\n",
      "\t\t 5  --  0.096891835\n",
      "\t\t 6  --  0.11747905\n",
      "\t\t 7  --  0.27705052\n",
      "\t\t 8  --  0.37569568\n",
      "\t\t 9  --  0.4899761\n",
      "\t\t 10  --  0.6233161\n",
      "1658 - 13 changed.\n",
      "--------------------------\n",
      "******  169  ********\n",
      "\t\t 0  --  9.56184e-05\n",
      "\t\t 1  --  0.0064158402\n",
      "\t\t 2  --  0.008715178\n",
      "\t\t 3  --  0.022567468\n",
      "\t\t 4  --  0.022567468\n",
      "\t\t 5  --  0.23735873\n",
      "\t\t 6  --  0.56350607\n",
      "1675 - 5 changed.\n",
      "--------------------------\n",
      "******  170  ********\n",
      "\t\t 0  --  0.00094555103\n",
      "\t\t 1  --  0.008636997\n",
      "\t\t 2  --  0.008636997\n",
      "\t\t 3  --  0.016667493\n",
      "\t\t 4  --  0.051745936\n",
      "\t\t 5  --  0.27254328\n",
      "\t\t 6  --  0.5062533\n",
      "1677 - 9 changed.\n",
      "--------------------------\n",
      "******  171  ********\n",
      "\t\t 0  --  0.23812245\n",
      "\t\t 1  --  0.8926534\n",
      "1684 - 3 changed.\n",
      "--------------------------\n",
      "******  172  ********\n",
      "\t\t 0  --  0.81377715\n",
      "1694 - 1 changed.\n",
      "--------------------------\n",
      "******  173  ********\n",
      "\t\t 0  --  0.8129592\n",
      "1706 - 1 changed.\n",
      "--------------------------\n",
      "******  174  ********\n",
      "\t\t 0  --  0.015524602\n",
      "\t\t 1  --  0.036091886\n",
      "\t\t 2  --  0.1716034\n",
      "\t\t 3  --  0.3740348\n",
      "\t\t 4  --  0.5914103\n",
      "1712 - 9 changed.\n",
      "--------------------------\n",
      "******  175  ********\n",
      "\t\t 0  --  0.49267253\n",
      "\t\t 1  --  0.8935582\n",
      "1716 - 3 changed.\n",
      "--------------------------\n",
      "******  176  ********\n",
      "\t\t 0  --  0.67165124\n",
      "1721 - 1 changed.\n",
      "--------------------------\n",
      "******  177  ********\n",
      "\t\t 0  --  0.00026638314\n",
      "\t\t 1  --  0.015685368\n",
      "\t\t 2  --  0.015685368\n",
      "\t\t 3  --  0.08705048\n",
      "\t\t 4  --  0.39986855\n",
      "\t\t 5  --  0.8466518\n",
      "1751 - 8 changed.\n",
      "--------------------------\n",
      "******  178  ********\n",
      "\t\t 0  --  0.32142153\n",
      "\t\t 1  --  0.5019398\n",
      "1755 - 2 changed.\n",
      "--------------------------\n",
      "******  179  ********\n",
      "\t\t 0  --  0.3259282\n",
      "\t\t 1  --  0.5550594\n",
      "1764 - 2 changed.\n",
      "--------------------------\n",
      "******  180  ********\n",
      "\t\t 0  --  0.35826942\n",
      "\t\t 1  --  0.543096\n",
      "1767 - 3 changed.\n",
      "--------------------------\n",
      "******  181  ********\n",
      "\t\t 0  --  0.7204405\n",
      "1769 - 1 changed.\n",
      "--------------------------\n",
      "******  182  ********\n",
      "\t\t 0  --  0.007969931\n",
      "\t\t 1  --  0.20641252\n",
      "\t\t 2  --  0.6978365\n",
      "1786 - 4 changed.\n",
      "--------------------------\n",
      "******  183  ********\n",
      "\t\t 0  --  0.006599674\n",
      "\t\t 1  --  0.045882534\n",
      "\t\t 2  --  0.06251921\n",
      "\t\t 3  --  0.32432044\n",
      "\t\t 4  --  0.54395217\n",
      "1797 - 7 changed.\n",
      "--------------------------\n",
      "******  184  ********\n",
      "\t\t 0  --  6.179906e-05\n",
      "\t\t 1  --  0.00043571813\n",
      "\t\t 2  --  0.0011406938\n",
      "\t\t 3  --  0.033328395\n",
      "\t\t 4  --  0.041783653\n",
      "\t\t 5  --  0.041783653\n",
      "\t\t 6  --  0.041783653\n",
      "\t\t 7  --  0.18758269\n",
      "\t\t 8  --  0.506468\n",
      "1802 - 8 changed.\n",
      "--------------------------\n",
      "******  185  ********\n",
      "\t\t 0  --  0.06755076\n",
      "\t\t 1  --  0.11572464\n",
      "\t\t 2  --  0.3421698\n",
      "\t\t 3  --  0.5874256\n",
      "1804 - 7 changed.\n",
      "--------------------------\n",
      "******  186  ********\n",
      "\t\t 0  --  0.010383041\n",
      "\t\t 1  --  0.16637217\n",
      "\t\t 2  --  0.7017095\n",
      "1809 - 5 changed.\n",
      "--------------------------\n",
      "******  187  ********\n",
      "\t\t 0  --  0.42135563\n",
      "\t\t 1  --  0.5256873\n",
      "1810 - 2 changed.\n",
      "--------------------------\n",
      "******  188  ********\n",
      "\t\t 0  --  0.16399035\n",
      "\t\t 1  --  0.210042\n",
      "\t\t 2  --  0.37276\n",
      "\t\t 3  --  0.7353898\n",
      "1828 - 2 changed.\n",
      "--------------------------\n",
      "******  189  ********\n",
      "\t\t 0  --  0.5305287\n",
      "1845 - 1 changed.\n",
      "--------------------------\n",
      "******  190  ********\n",
      "\t\t 0  --  0.7331186\n",
      "1854 - 1 changed.\n",
      "--------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "******  191  ********\n",
      "\t\t 0  --  0.002563564\n",
      "\t\t 1  --  0.0055873785\n",
      "\t\t 2  --  0.0059753605\n",
      "\t\t 3  --  0.007867068\n",
      "\t\t 4  --  0.013503993\n",
      "\t\t 5  --  0.0135335745\n",
      "\t\t 6  --  0.018860644\n",
      "\t\t 7  --  0.02545974\n",
      "\t\t 8  --  0.03287588\n",
      "\t\t 9  --  0.03287588\n",
      "\t\t 10  --  0.046571806\n",
      "\t\t 11  --  0.070145756\n",
      "\t\t 12  --  0.08857982\n",
      "\t\t 13  --  0.08857982\n",
      "\t\t 14  --  0.10529671\n",
      "\t\t 15  --  0.1344016\n",
      "\t\t 16  --  0.16498345\n",
      "\t\t 17  --  0.16498345\n",
      "\t\t 18  --  0.21268459\n",
      "\t\t 19  --  0.21677099\n",
      "\t\t 20  --  0.21677099\n",
      "\t\t 21  --  0.22076964\n",
      "\t\t 22  --  0.23354492\n",
      "\t\t 23  --  0.2546097\n",
      "\t\t 24  --  0.29033723\n",
      "\t\t 25  --  0.29033723\n",
      "\t\t 26  --  0.31929314\n",
      "\t\t 27  --  0.31929314\n",
      "\t\t 28  --  0.71250224\n",
      "1873 - 19 changed.\n",
      "--------------------------\n",
      "******  192  ********\n",
      "\t\t 0  --  0.59134144\n",
      "1874 - 1 changed.\n",
      "--------------------------\n",
      "******  193  ********\n",
      "\t\t 0  --  0.41071123\n",
      "\t\t 1  --  0.8056726\n",
      "1880 - 2 changed.\n",
      "--------------------------\n",
      "******  194  ********\n",
      "\t\t 0  --  0.464266\n",
      "\t\t 1  --  0.8201281\n",
      "1886 - 2 changed.\n",
      "--------------------------\n",
      "******  195  ********\n",
      "\t\t 0  --  0.027469067\n",
      "\t\t 1  --  0.12020541\n",
      "\t\t 2  --  0.4979116\n",
      "\t\t 3  --  0.65397406\n",
      "1924 - 4 changed.\n",
      "--------------------------\n",
      "******  196  ********\n",
      "\t\t 0  --  0.013922011\n",
      "\t\t 1  --  0.03587096\n",
      "\t\t 2  --  0.07439979\n",
      "\t\t 3  --  0.2527352\n",
      "\t\t 4  --  0.4965154\n",
      "\t\t 5  --  0.96593463\n",
      "1937 - 8 changed.\n",
      "--------------------------\n",
      "******  197  ********\n",
      "\t\t 0  --  0.010176303\n",
      "\t\t 1  --  0.019567747\n",
      "\t\t 2  --  0.5735331\n",
      "1944 - 3 changed.\n",
      "--------------------------\n",
      "******  198  ********\n",
      "\t\t 0  --  0.013274136\n",
      "\t\t 1  --  0.04447436\n",
      "\t\t 2  --  0.20700437\n",
      "\t\t 3  --  0.90491897\n",
      "1945 - 6 changed.\n",
      "--------------------------\n",
      "******  199  ********\n",
      "\t\t 0  --  0.03691605\n",
      "\t\t 1  --  0.042647354\n",
      "\t\t 2  --  0.0654682\n",
      "\t\t 3  --  0.22782595\n",
      "\t\t 4  --  0.22782595\n",
      "\t\t 5  --  0.47506717\n",
      "\t\t 6  --  0.688321\n",
      "1949 - 12 changed.\n",
      "--------------------------\n",
      "******  200  ********\n",
      "\t\t 0  --  0.028079277\n",
      "\t\t 1  --  0.67549986\n",
      "1960 - 3 changed.\n",
      "--------------------------\n"
     ]
    }
   ],
   "source": [
    "SAMPLE_SIZE = 5000\n",
    "TEST_SIZE = 200\n",
    "test_idx = np.random.choice(len(dataset.test_y), SAMPLE_SIZE, replace=False)\n",
    "test_len = []\n",
    "for i in range(SAMPLE_SIZE):\n",
    "    test_len.append(len(dataset.test_seqs2[test_idx[i]]))\n",
    "print('Shortest sentence in our test set is %d words' %np.min(test_len))\n",
    "\n",
    "test_list = []\n",
    "orig_list = []\n",
    "orig_label_list = []\n",
    "adv_list = []\n",
    "dist_list = []\n",
    "\n",
    "for i in range(SAMPLE_SIZE):\n",
    "    x_orig = test_x[test_idx[i]]\n",
    "    orig_label = test_y[test_idx[i]]\n",
    "    orig_preds=  model.predict(sess, x_orig[np.newaxis, :])[0]\n",
    "    # print(orig_label, orig_preds, np.argmax(orig_preds))\n",
    "    if np.argmax(orig_preds) != orig_label:\n",
    "        #print('skipping wrong classifed ..')\n",
    "        #print('--------------------------')\n",
    "        continue\n",
    "    x_len = np.sum(np.sign(x_orig))\n",
    "    if x_len >= 100:\n",
    "        #print('skipping too long input..')\n",
    "        #print('--------------------------')\n",
    "        continue\n",
    "    # if np.max(orig_preds) < 0.90:\n",
    "    #    print('skipping low confidence .. \\n-----\\n')\n",
    "    #    continue\n",
    "    print('****** ', len(test_list) + 1, ' ********')\n",
    "    test_list.append(test_idx[i])\n",
    "    orig_list.append(x_orig)\n",
    "    target_label = 1 if orig_label == 0 else 0\n",
    "    orig_label_list.append(orig_label)\n",
    "    x_adv = ga_atttack.attack( x_orig, target_label)\n",
    "    adv_list.append(x_adv)\n",
    "    if x_adv is None:\n",
    "        print('%d failed' %(i+1))\n",
    "        dist_list.append(100000)\n",
    "    else:\n",
    "        num_changes = np.sum(x_orig != x_adv)\n",
    "        print('%d - %d changed.' %(i+1, num_changes))\n",
    "        dist_list.append(num_changes)\n",
    "        # display_utils.visualize_attack(sess, model, dataset, x_orig, x_adv)\n",
    "    print('--------------------------')\n",
    "    if (len(test_list)>= TEST_SIZE):\n",
    "        break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Compute Attack success rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "orig_len = [np.sum(np.sign(x)) for x in orig_list]\n",
    "normalized_dist_list = [dist_list[i]/orig_len[i] for i in range(len(orig_list)) ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attack success rate : 92.00%\n",
      "Median percentange of modifications: 6.45% \n",
      "Mean percentange of modifications: 9.14% \n"
     ]
    }
   ],
   "source": [
    "SUCCESS_THRESHOLD  = 0.25\n",
    "successful_attacks = [x < SUCCESS_THRESHOLD for x in normalized_dist_list]\n",
    "print('Attack success rate : {:.2f}%'.format(np.mean(successful_attacks)*100))\n",
    "print('Median percentange of modifications: {:.02f}% '.format(\n",
    "    np.median([x for x in normalized_dist_list if x < 1])*100))\n",
    "print('Mean percentange of modifications: {:.02f}% '.format(\n",
    "    np.mean([x for x in normalized_dist_list if x < 1])*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original Prediction = Negative. (Confidence = 99.81) \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "poorly directed short film <b style='color:green'>shot</b> on hi def or betacam it appears it <b style='color:green'>screams</b> student film video all the way the <b style='color:green'>premise</b> is limited in scope and the short actually feels a lot <b style='color:green'>longer</b> than it runs some interesting acting moments and some decent production value but not enough to lift this film from the hole it has fallen <b style='color:green'>into</b>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------  After attack -------------\n",
      "New Prediction = Positive. (Confidence = 59.80) \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "poorly directed short film <b style='color:red'>gunshot</b> on hi def or betacam it appears it <b style='color:red'>shrieks</b> student film video all the way the <b style='color:red'>premises</b> is limited in scope and the short actually feels a lot <b style='color:red'>longest</b> than it runs some interesting acting moments and some decent production value but not enough to lift this film from the hole it has fallen <b style='color:red'>in</b>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "visual_idx = np.random.choice(len(orig_list))\n",
    "display_utils.visualize_attack(sess, model, dataset, orig_list[visual_idx], adv_list[visual_idx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original Prediction = Positive. (Confidence = 85.73) \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "absolutely fantastic whatever i say wouldn't do this <b style='color:green'>underrated</b> movie the justice it <b style='color:green'>deserves</b> watch it now fantastic"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------  After attack -------------\n",
      "New Prediction = Negative. (Confidence = 50.19) \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "absolutely fantastic whatever i say wouldn't do this <b style='color:red'>underestimated</b> movie the justice it <b style='color:red'>deserve</b> watch it now fantastic"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "visual_idx = np.random.choice(len(orig_list))\n",
    "display_utils.visualize_attack(sess, model, dataset, orig_list[visual_idx], adv_list[visual_idx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Save success\n",
    "with open('attack_results_final.pkl', 'wb') as f:\n",
    "    pickle.dump((test_list, orig_list, orig_label_list, adv_list, normalized_dist_list), f)\n",
    "    \n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}