Link Prediction-Copy1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import numpy as np\n",
    "import hnswlib\n",
    "import networkx as nx\n",
    "from collections import defaultdict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import scipy.sparse as sp\n",
    "\n",
    "\"\"\"\n",
    "Disclaimer: functions defined from lines 15 to 36 in this file come from \n",
    "tkipf/gae original repository on Graph Autoencoders. Moreover, the\n",
    "mask_test_edges function is borrowed from philipjackson's mask_test_edges \n",
    "pull request on this same repository.\n",
    "\"\"\"\n",
    "\n",
    "def sparse_to_tuple(sparse_mx):\n",
    "    if not sp.isspmatrix_coo(sparse_mx):\n",
    "        sparse_mx = sparse_mx.tocoo()\n",
    "    coords = np.vstack((sparse_mx.row, sparse_mx.col)).transpose()\n",
    "    values = sparse_mx.data\n",
    "    shape = sparse_mx.shape\n",
    "    return coords, values, shape\n",
    "\n",
    "def preprocess_graph(adj):\n",
    "    adj = sp.coo_matrix(adj)\n",
    "    adj_ = adj + sp.eye(adj.shape[0])\n",
    "    degree_mat_inv_sqrt = sp.diags(np.power(np.array(adj_.sum(1)), -0.5).flatten())\n",
    "    adj_normalized = adj_.dot(degree_mat_inv_sqrt).transpose().dot(degree_mat_inv_sqrt)\n",
    "    return sparse_to_tuple(adj_normalized)\n",
    "\n",
    "def construct_feed_dict(adj_normalized, adj, features, placeholders):\n",
    "    # Construct feed dictionary\n",
    "    feed_dict = dict()\n",
    "    feed_dict.update({placeholders['features']: features})\n",
    "    feed_dict.update({placeholders['adj']: adj_normalized})\n",
    "    feed_dict.update({placeholders['adj_orig']: adj})\n",
    "    return feed_dict\n",
    "\n",
    "def mask_test_edges(adj, test_percent=1., val_percent=0.):\n",
    "    \"\"\" Randomly removes some edges from original graph to create\n",
    "    test and validation sets for link prediction task\n",
    "    :param adj: complete sparse adjacency matrix of the graph\n",
    "    :param test_percent: percentage of edges in test set\n",
    "    :param val_percent: percentage of edges in validation set\n",
    "    :return: train incomplete adjacency matrix, validation and test sets\n",
    "    \"\"\"\n",
    "    # Remove diagonal elements\n",
    "    adj = adj - sp.dia_matrix((adj.diagonal()[None, :], [0]), shape=adj.shape)\n",
    "    adj.eliminate_zeros()\n",
    "    # Check that diag is zero:\n",
    "    #assert adj.diagonal().sum() == 0\n",
    "\n",
    "    edges_positive, _, _ = sparse_to_tuple(adj)\n",
    "    # Filtering out edges from lower triangle of adjacency matrix\n",
    "    edges_positive = edges_positive[edges_positive[:,1] > edges_positive[:,0],:]\n",
    "    # val_edges, val_edges_false, test_edges, test_edges_false = None, None, None, None\n",
    "\n",
    "    # number of positive (and negative) edges in test and val sets:\n",
    "    num_test = int(np.floor(edges_positive.shape[0] / (100. / test_percent)))\n",
    "    num_val = 0\n",
    "\n",
    "    # sample positive edges for test and val sets:\n",
    "    edges_positive_idx = np.arange(edges_positive.shape[0])\n",
    "    np.random.shuffle(edges_positive_idx)\n",
    "    val_edge_idx = edges_positive_idx[:num_val]\n",
    "    test_edge_idx = edges_positive_idx[num_val:(num_val + num_test)]\n",
    "    test_edges = edges_positive[test_edge_idx] # positive test edges\n",
    "    val_edges = edges_positive[val_edge_idx] # positive val edges\n",
    "    train_edges = np.delete(edges_positive, np.hstack([test_edge_idx, val_edge_idx]), axis = 0) # positive train edges\n",
    "\n",
    "    # the above strategy for sampling without replacement will not work for\n",
    "    # sampling negative edges on large graphs, because the pool of negative\n",
    "    # edges is much much larger due to sparsity, therefore we'll use\n",
    "    # the following strategy:\n",
    "    # 1. sample random linear indices from adjacency matrix WITH REPLACEMENT\n",
    "    # (without replacement is super slow). sample more than we need so we'll\n",
    "    # probably have enough after all the filtering steps.\n",
    "    # 2. remove any edges that have already been added to the other edge lists\n",
    "    # 3. convert to (i,j) coordinates\n",
    "    # 4. swap i and j where i > j, to ensure they're upper triangle elements\n",
    "    # 5. remove any duplicate elements if there are any\n",
    "    # 6. remove any diagonal elements\n",
    "    # 7. if we don't have enough edges, repeat this process until we get enough\n",
    "    positive_idx, _, _ = sparse_to_tuple(adj) # [i,j] coord pairs for all true edges\n",
    "    positive_idx = positive_idx[:,0]*adj.shape[0] + positive_idx[:,1] # linear indices\n",
    "    test_edges_false = np.empty((0,2),dtype='int64')\n",
    "    idx_test_edges_false = np.empty((0,),dtype='int64')\n",
    "\n",
    "    while len(test_edges_false) < len(test_edges):\n",
    "        # step 1:\n",
    "        idx = np.random.choice(adj.shape[0]**2, 2*(num_test - len(test_edges_false)), replace = True)\n",
    "        # step 2:\n",
    "        idx = idx[~np.in1d(idx, positive_idx, assume_unique = True)]\n",
    "        idx = idx[~np.in1d(idx, idx_test_edges_false, assume_unique = True)]\n",
    "        # step 3:\n",
    "        rowidx = idx // adj.shape[0]\n",
    "        colidx = idx % adj.shape[0]\n",
    "        coords = np.vstack((rowidx,colidx)).transpose()\n",
    "        # step 4:\n",
    "        lowertrimask = coords[:,0] > coords[:,1]\n",
    "        coords[lowertrimask] = coords[lowertrimask][:,::-1]\n",
    "        # step 5:\n",
    "        coords = np.unique(coords, axis = 0) # note: coords are now sorted lexicographically\n",
    "        np.random.shuffle(coords) # not anymore\n",
    "        # step 6:\n",
    "        coords = coords[coords[:,0] != coords[:,1]]\n",
    "        # step 7:\n",
    "        coords = coords[:min(num_test, len(idx))]\n",
    "        test_edges_false = np.append(test_edges_false, coords, axis = 0)\n",
    "        idx = idx[:min(num_test, len(idx))]\n",
    "        idx_test_edges_false = np.append(idx_test_edges_false, idx)\n",
    "\n",
    "    val_edges_false = np.empty((0,2), dtype = 'int64')\n",
    "    idx_val_edges_false = np.empty((0,), dtype = 'int64')\n",
    "#     while len(val_edges_false) < len(val_edges):\n",
    "#         # step 1:\n",
    "#         idx = np.random.choice(adj.shape[0]**2, 2*(num_val - len(val_edges_false)), replace = True)\n",
    "#         # step 2:\n",
    "#         idx = idx[~np.in1d(idx, positive_idx, assume_unique = True)]\n",
    "#         idx = idx[~np.in1d(idx, idx_test_edges_false, assume_unique = True)]\n",
    "#         idx = idx[~np.in1d(idx, idx_val_edges_false, assume_unique = True)]\n",
    "#         # step 3:\n",
    "#         rowidx = idx // adj.shape[0]\n",
    "#         colidx = idx % adj.shape[0]\n",
    "#         coords = np.vstack((rowidx,colidx)).transpose()\n",
    "#         # step 4:\n",
    "#         lowertrimask = coords[:,0] > coords[:,1]\n",
    "#         coords[lowertrimask] = coords[lowertrimask][:,::-1]\n",
    "#         # step 5:\n",
    "#         coords = np.unique(coords, axis = 0) # note: coords are now sorted lexicographically\n",
    "#         np.random.shuffle(coords) # not any more\n",
    "#         # step 6:\n",
    "#         coords = coords[coords[:,0] != coords[:,1]]\n",
    "#         # step 7:\n",
    "#         coords = coords[:min(num_val, len(idx))]\n",
    "#         val_edges_false = np.append(val_edges_false, coords, axis = 0)\n",
    "#         idx = idx[:min(num_val, len(idx))]\n",
    "#         idx_val_edges_false = np.append(idx_val_edges_false, idx)\n",
    "\n",
    "    # sanity checks:\n",
    "#     train_edges_linear = train_edges[:,0]*adj.shape[0] + train_edges[:,1]\n",
    "#     test_edges_linear = test_edges[:,0]*adj.shape[0] + test_edges[:,1]\n",
    "#     assert not np.any(np.in1d(idx_test_edges_false, positive_idx))\n",
    "#     assert not np.any(np.in1d(idx_val_edges_false, positive_idx))\n",
    "#     assert not np.any(np.in1d(val_edges[:,0]*adj.shape[0]+val_edges[:,1], train_edges_linear))\n",
    "#     assert not np.any(np.in1d(test_edges_linear, train_edges_linear))\n",
    "#     assert not np.any(np.in1d(val_edges[:,0]*adj.shape[0]+val_edges[:,1], test_edges_linear))\n",
    "\n",
    "    # Re-build adj matrix\n",
    "    data = np.ones(train_edges.shape[0])\n",
    "    adj_train = sp.csr_matrix((data, (train_edges[:, 0], train_edges[:, 1])), shape=adj.shape)\n",
    "    adj_train = adj_train + adj_train.T\n",
    "    return adj_train, val_edges, val_edges_false, test_edges, test_edges_false"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reading Embeddings and creating Indexer for NN search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "IDList = []                                # List of paper IDs\n",
    "NNList = []                                # List of list, NNList[i]: NNs to paper whose id is IDList[i]\n",
    "embeddings = []                            # Embeddings read from the input file\n",
    "\n",
    "with open('./data/dblp_Abstract_2Thresholded_USE_Trans_Embeddings.json', 'r') as file:\n",
    "    for line in file:\n",
    "        data = json.loads(line)\n",
    "        paperID = data['id']\n",
    "        embedding = data['embedding']\n",
    "        IDList.append(paperID)\n",
    "        embeddings.append(embedding)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "numElements = len(IDList)\n",
    "dimension = len(embeddings[0])\n",
    "embeddings = np.asarray(embeddings)\n",
    "data_labels = np.arange(numElements)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "p = hnswlib.Index(space = 'cosine', dim = dimension) # possible options are l2, cosine or ip\n",
    "\n",
    "# Initing index - the maximum number of elements should be known beforehand\n",
    "p.init_index(max_elements = numElements, ef_construction = 200, M = 20)\n",
    "\n",
    "# Element insertion (can be called several times):\n",
    "p.add_items(embeddings, data_labels)\n",
    "\n",
    "# Controlling the recall by setting ef:\n",
    "p.set_ef(50) # ef should always be > k\n",
    "\n",
    "# Query dataset, k - number of closest elements (returns 2 numpy arrays)\n",
    "labels, _ = p.knn_query(embeddings, k = 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "index_path='./models/USETranshnswlib.bin'\n",
    "print(\"Saving index to '%s'\" % index_path)\n",
    "p.save_index(\"./models/USETranshnswlib.bin\")\n",
    "del p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "p = hnswlib.Index(space='cosine', dim=dimension)  # the space can be changed - keeps the data, alters the distance function.\n",
    "\n",
    "# Increase the total capacity (max_elements), so that it will handle the new data\n",
    "p.load_index(\"./models/USETranshnswlib.bin\", max_elements = numElements)\n",
    "labels, _ = p.knn_query(embeddings, k = 4)\n",
    "del p\n",
    "del embeddings\n",
    "del data_labels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Examples of NN obtained using the Text Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "titles = []\n",
    "with open('./data/dblp_AIpapers2Thresholded.json', 'r') as file:\n",
    "    for line in file:\n",
    "        data = json.loads(line)\n",
    "        titles.append(data['title'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "count = 5\n",
    "for i in range(count):\n",
    "    print('Paper: ', titles[i])\n",
    "    print('Nearest Papers: ', [titles[ind] for ind in labels[i] if ind != i])\n",
    "    print('\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Building Adjacency List for Node Embeddings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating Citation Adjacency List"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "adjList = defaultdict(set)                          # Convert set to list later for node2vec, set: to handle duplicates\n",
    "with open('./data/dblp_AIpapers2Thresholded.json', 'r') as file:\n",
    "    for line in file:\n",
    "        data = json.loads(line)\n",
    "        paperID = data['id']\n",
    "        references = data.get('references', [])\n",
    "        for referencedPaper in references:\n",
    "            adjList[paperID].add(referencedPaper)\n",
    "            adjList[referencedPaper].add(paperID)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Augmenting Adj List with FastText NNs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nnToKeep = 4\n",
    "id = 0\n",
    "for label in labels:\n",
    "    paperID = IDList[id]\n",
    "    label = [IDList[index] for index in label if index != id]\n",
    "    if (len(label) > nnToKeep):\n",
    "        del label[nnToKeep:]\n",
    "    for referencedPaper in label:\n",
    "        adjList[paperID].add(referencedPaper)\n",
    "        adjList[referencedPaper].add(paperID)\n",
    "    id += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating NetworkX Graph and reporting graph statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of nodes:  471633 . Number of edges:  5464345 . Avg Degree:  23.172021465843144\n"
     ]
    }
   ],
   "source": [
    "adjList = {key: list(values) for key, values in adjList.items()}\n",
    "G = nx.from_dict_of_lists(adjList)\n",
    "\n",
    "nnodes = G.number_of_nodes()\n",
    "avgDegree = sum(d for n, d in G.degree()) / float(nnodes)\n",
    "print('Number of nodes: ', nnodes, '. Number of edges: ', G.number_of_edges(), '. Avg Degree: ', avgDegree)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "adj_sparse = nx.to_scipy_sparse_matrix(G)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Constructing new graph\n"
     ]
    }
   ],
   "source": [
    "# Perform train-test split\n",
    "adj_train, val_edges, val_edges_false, test_edges, test_edges_false = mask_test_edges(adj_sparse)\n",
    "print('Constructing new graph')\n",
    "G_train = nx.from_scipy_sparse_matrix(adj_train) # new graph object with only non-hidden edges"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "G_train.number_of_nodes() == G.number_of_nodes()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "nodes = list(G.nodes())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "471633"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(nodes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found\n"
     ]
    }
   ],
   "source": [
    "for adjV in adjList[u]:\n",
    "    if adjV == v:\n",
    "        print('Found')\n",
    "    \n",
    "## paperID mapping: G.nodes() are paperIDs one to one mapped with G_train.nodes which are 0 to |V| - 1\n",
    "## test edges and test edges negative are integer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Node2Vec Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Computing transition probabilities:   4%|▍         | 18408/471633 [27:11<12:10:31, 10.34it/s]"
     ]
    }
   ],
   "source": [
    "from node2vec import Node2Vec\n",
    "walkLength = 6\n",
    "node2vec = Node2Vec(G_train, walk_length = walkLength)#, workers = 12, temp_folder = './data/tmp_data')\n",
    "          "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = node2vec.fit()  # returns a gensim wv model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "outFileName = './models/node2vec_USE_2Citation_Embeddings_WL_' + str(walkLength) + '_NN_' + str(nnToKeep) + '.kv'\n",
    "model.wv.save_word2vec_format(outFileName)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Edge Prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getEdgeEmbedding(embedding1, embedding2, policy='Hadamard'):\n",
    "    if (policy=='Hadamard'):\n",
    "        return embedding1 * embedding2\n",
    "    elif(policy=='Avg'):\n",
    "        return (embedding1 + embedding2) / 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def average(lis):\n",
    "    return (sum(lis) / len(lis))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "word2VecMode = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddingDict = dict()\n",
    "\n",
    "if (word2VecMode):\n",
    "    embeddingFileName = './data/dblpAbstract_2Thresholded_FT_Embeddings.json'\n",
    "    with open(embeddingFileName, 'r') as file:\n",
    "        for line in file:\n",
    "            data = json.loads(line)\n",
    "            embeddingDict[data['id']] = data['embedding']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'embeddingDict' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-37-0af9b6b86b6e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     13\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0medge\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m             \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0membedding1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0membeddingDict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpaperID1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0membedding2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0membeddingDict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpaperID2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'embeddingDict' is not defined"
     ]
    }
   ],
   "source": [
    "            \n",
    "X = []\n",
    "edges = [*test_edges, *test_edges_false]\n",
    "for edge in edges :\n",
    "    if (word2VecMode):\n",
    "        try:\n",
    "            paperID1 = nodes[edge[0]]\n",
    "        except:\n",
    "            print(edge[0])\n",
    "            break\n",
    "        try:\n",
    "            paperID2 = nodes[edge[1]]\n",
    "        except:\n",
    "            print(edge[1])\n",
    "            break\n",
    "        embedding1 = np.asarray(embeddingDict[paperID1])\n",
    "        embedding2 = np.asarray(embeddingDict[paperID2])\n",
    "    else:\n",
    "        if edge[0] in model.wv.vocab:\n",
    "            embedding1 = np.asarray(model.wv[edge[0]])\n",
    "        else:\n",
    "            embedding1 = np.asarray([0] * 128)\n",
    "        if edge[1] in model.wv.vocab:\n",
    "            embedding2 = np.asarray(model.wv[edge[1]])\n",
    "        else:\n",
    "            embedding2 = np.asarray([0] * 128)\n",
    "    edgeEmbedding =  getEdgeEmbedding(embedding1, embedding2)\n",
    "    X.append(edgeEmbedding)\n",
    "\n",
    "Y = np.concatenate(np.ones(len(test_edges), dtype = int), np.zeros(len(test_edges_false), dtype=int))\n",
    "if (word2VecMode):\n",
    "    del embeddingDict    \n",
    "X = np.asarray(X)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.gaussian_process.kernels import RBF\n",
    "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, BaggingClassifier\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "\n",
    "\n",
    "names = [\n",
    " \"Random Forest\", \"Neural Net\", \"Logistic Regression\", \"Linear SVC\" ]\n",
    "\n",
    "classifiers = [\n",
    "    RandomForestClassifier(verbose=True, n_jobs = -1),\n",
    "    MLPClassifier(verbose=True, early_stopping=True),\n",
    "    AdaBoostClassifier(),\n",
    "    OneVsRestClassifier(BaggingClassifier(LinearSVC(),n_jobs = -1))]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting:  1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 48 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   15.4s finished\n",
      "[Parallel(n_jobs=48)]: Using backend ThreadingBackend with 48 concurrent workers.\n",
      "[Parallel(n_jobs=48)]: Done 100 out of 100 | elapsed:    0.1s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count  1\n",
      "Fitting:  2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 48 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   15.0s finished\n",
      "[Parallel(n_jobs=48)]: Using backend ThreadingBackend with 48 concurrent workers.\n",
      "[Parallel(n_jobs=48)]: Done 100 out of 100 | elapsed:    0.1s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count  2\n",
      "Fitting:  3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 48 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   15.6s finished\n",
      "[Parallel(n_jobs=48)]: Using backend ThreadingBackend with 48 concurrent workers.\n",
      "[Parallel(n_jobs=48)]: Done 100 out of 100 | elapsed:    0.1s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count  3\n",
      "Fitting:  4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 48 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   15.4s finished\n",
      "[Parallel(n_jobs=48)]: Using backend ThreadingBackend with 48 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count  4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=48)]: Done 100 out of 100 | elapsed:    0.1s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting:  5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 48 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   14.9s finished\n",
      "[Parallel(n_jobs=48)]: Using backend ThreadingBackend with 48 concurrent workers.\n",
      "[Parallel(n_jobs=48)]: Done 100 out of 100 | elapsed:    0.1s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count  5\n",
      "Name Random Forest . Avg Precision:  0.7716315809855593 . Avg Recall:  0.769714307624117 . Avg F-1 Score:  0.7693101532464279\n",
      "Fitting:  1\n",
      "Iteration 1, loss = 0.69222773\n",
      "Validation score: 0.500629\n",
      "Iteration 2, loss = 0.68977166\n",
      "Validation score: 0.559190\n",
      "Iteration 3, loss = 0.68747817\n",
      "Validation score: 0.631477\n",
      "Iteration 4, loss = 0.68475911\n",
      "Validation score: 0.589157\n",
      "Iteration 5, loss = 0.68124036\n",
      "Validation score: 0.679744\n",
      "Iteration 6, loss = 0.67625042\n",
      "Validation score: 0.659156\n",
      "Iteration 7, loss = 0.66971378\n",
      "Validation score: 0.693355\n",
      "Iteration 8, loss = 0.66221698\n",
      "Validation score: 0.698502\n",
      "Iteration 9, loss = 0.65353430\n",
      "Validation score: 0.698502\n",
      "Iteration 10, loss = 0.64473392\n",
      "Validation score: 0.699188\n",
      "Iteration 11, loss = 0.63625275\n",
      "Validation score: 0.712456\n",
      "Iteration 12, loss = 0.62788696\n",
      "Validation score: 0.716116\n",
      "Iteration 13, loss = 0.61963378\n",
      "Validation score: 0.708567\n",
      "Iteration 14, loss = 0.61152296\n",
      "Validation score: 0.727325\n",
      "Iteration 15, loss = 0.60445811\n",
      "Validation score: 0.728697\n",
      "Iteration 16, loss = 0.59695007\n",
      "Validation score: 0.726638\n",
      "Iteration 17, loss = 0.59037385\n",
      "Validation score: 0.741050\n",
      "Iteration 18, loss = 0.58445142\n",
      "Validation score: 0.732472\n",
      "Iteration 19, loss = 0.57851797\n",
      "Validation score: 0.747569\n",
      "Iteration 20, loss = 0.57271481\n",
      "Validation score: 0.751916\n",
      "Iteration 21, loss = 0.56792399\n",
      "Validation score: 0.754203\n",
      "Iteration 22, loss = 0.56309967\n",
      "Validation score: 0.757177\n",
      "Iteration 23, loss = 0.55845604\n",
      "Validation score: 0.760151\n",
      "Iteration 24, loss = 0.55457528\n",
      "Validation score: 0.761180\n",
      "Iteration 25, loss = 0.55095063\n",
      "Validation score: 0.759350\n",
      "Iteration 26, loss = 0.54699949\n",
      "Validation score: 0.761295\n",
      "Iteration 27, loss = 0.54425942\n",
      "Validation score: 0.762896\n",
      "Iteration 28, loss = 0.54069864\n",
      "Validation score: 0.763582\n",
      "Iteration 29, loss = 0.53794307\n",
      "Validation score: 0.769873\n",
      "Iteration 30, loss = 0.53487450\n",
      "Validation score: 0.766556\n",
      "Iteration 31, loss = 0.53242046\n",
      "Validation score: 0.766213\n",
      "Iteration 32, loss = 0.53005544\n",
      "Validation score: 0.770674\n",
      "Iteration 33, loss = 0.52779391\n",
      "Validation score: 0.773304\n",
      "Iteration 34, loss = 0.52542358\n",
      "Validation score: 0.774677\n",
      "Iteration 35, loss = 0.52345653\n",
      "Validation score: 0.773076\n",
      "Iteration 36, loss = 0.52160817\n",
      "Validation score: 0.774219\n",
      "Iteration 37, loss = 0.52009443\n",
      "Validation score: 0.769873\n",
      "Iteration 38, loss = 0.51821757\n",
      "Validation score: 0.774906\n",
      "Iteration 39, loss = 0.51626573\n",
      "Validation score: 0.777879\n",
      "Iteration 40, loss = 0.51482237\n",
      "Validation score: 0.777879\n",
      "Iteration 41, loss = 0.51357743\n",
      "Validation score: 0.778680\n",
      "Iteration 42, loss = 0.51212744\n",
      "Validation score: 0.778794\n",
      "Iteration 43, loss = 0.51061324\n",
      "Validation score: 0.779366\n",
      "Iteration 44, loss = 0.50943467\n",
      "Validation score: 0.780167\n",
      "Iteration 45, loss = 0.50843207\n",
      "Validation score: 0.781196\n",
      "Iteration 46, loss = 0.50700408\n",
      "Validation score: 0.779252\n",
      "Iteration 47, loss = 0.50573701\n",
      "Validation score: 0.774906\n",
      "Iteration 48, loss = 0.50524982\n",
      "Validation score: 0.780968\n",
      "Iteration 49, loss = 0.50369373\n",
      "Validation score: 0.780968\n",
      "Iteration 50, loss = 0.50295971\n",
      "Validation score: 0.783713\n",
      "Iteration 51, loss = 0.50220508\n",
      "Validation score: 0.782912\n",
      "Iteration 52, loss = 0.50118622\n",
      "Validation score: 0.781768\n",
      "Iteration 53, loss = 0.49990392\n",
      "Validation score: 0.773991\n",
      "Iteration 54, loss = 0.49966485\n",
      "Validation score: 0.783255\n",
      "Iteration 55, loss = 0.49827835\n",
      "Validation score: 0.782683\n",
      "Iteration 56, loss = 0.49759384\n",
      "Validation score: 0.781425\n",
      "Iteration 57, loss = 0.49727846\n",
      "Validation score: 0.783941\n",
      "Iteration 58, loss = 0.49738097\n",
      "Validation score: 0.782569\n",
      "Iteration 59, loss = 0.49616755\n",
      "Validation score: 0.783370\n",
      "Iteration 60, loss = 0.49481614\n",
      "Validation score: 0.786115\n",
      "Iteration 61, loss = 0.49427572\n",
      "Validation score: 0.783827\n",
      "Iteration 62, loss = 0.49385913\n",
      "Validation score: 0.784513\n",
      "Iteration 63, loss = 0.49266558\n",
      "Validation score: 0.780510\n",
      "Iteration 64, loss = 0.49207980\n",
      "Validation score: 0.784170\n",
      "Iteration 65, loss = 0.49260731\n",
      "Validation score: 0.787602\n",
      "Iteration 66, loss = 0.49124071\n",
      "Validation score: 0.780510\n",
      "Iteration 67, loss = 0.49039744\n",
      "Validation score: 0.785314\n",
      "Iteration 68, loss = 0.49007288\n",
      "Validation score: 0.781997\n",
      "Iteration 69, loss = 0.48940741\n",
      "Validation score: 0.788974\n",
      "Iteration 70, loss = 0.48873367\n",
      "Validation score: 0.784399\n",
      "Iteration 71, loss = 0.48838922\n",
      "Validation score: 0.786343\n",
      "Iteration 72, loss = 0.48784533\n",
      "Validation score: 0.789889\n",
      "Iteration 73, loss = 0.48733679\n",
      "Validation score: 0.790118\n",
      "Iteration 74, loss = 0.48697393\n",
      "Validation score: 0.783370\n",
      "Iteration 75, loss = 0.48635319\n",
      "Validation score: 0.789317\n",
      "Iteration 76, loss = 0.48530332\n",
      "Validation score: 0.779709\n",
      "Iteration 77, loss = 0.48524085\n",
      "Validation score: 0.785085\n",
      "Iteration 78, loss = 0.48548540\n",
      "Validation score: 0.790118\n",
      "Iteration 79, loss = 0.48424616\n",
      "Validation score: 0.779938\n",
      "Iteration 80, loss = 0.48429932\n",
      "Validation score: 0.790461\n",
      "Iteration 81, loss = 0.48364710\n",
      "Validation score: 0.791719\n",
      "Iteration 82, loss = 0.48397704\n",
      "Validation score: 0.791376\n",
      "Iteration 83, loss = 0.48288400\n",
      "Validation score: 0.791490\n",
      "Iteration 84, loss = 0.48202020\n",
      "Validation score: 0.787144\n",
      "Iteration 85, loss = 0.48197702\n",
      "Validation score: 0.790575\n",
      "Iteration 86, loss = 0.48168187\n",
      "Validation score: 0.789203\n",
      "Iteration 87, loss = 0.48158034\n",
      "Validation score: 0.787258\n",
      "Iteration 88, loss = 0.48100442\n",
      "Validation score: 0.790575\n",
      "Iteration 89, loss = 0.48041724\n",
      "Validation score: 0.787144\n",
      "Iteration 90, loss = 0.47999014\n",
      "Validation score: 0.790118\n",
      "Iteration 91, loss = 0.47933011\n",
      "Validation score: 0.788745\n",
      "Iteration 92, loss = 0.48092344\n",
      "Validation score: 0.792177\n",
      "Iteration 93, loss = 0.47892985\n",
      "Validation score: 0.792291\n",
      "Iteration 94, loss = 0.47984764\n",
      "Validation score: 0.792062\n",
      "Iteration 95, loss = 0.47860719\n",
      "Validation score: 0.792177\n",
      "Iteration 96, loss = 0.47848261\n",
      "Validation score: 0.783941\n",
      "Iteration 97, loss = 0.47829157\n",
      "Validation score: 0.788974\n",
      "Iteration 98, loss = 0.47802976\n",
      "Validation score: 0.792520\n",
      "Iteration 99, loss = 0.47786859\n",
      "Validation score: 0.788860\n",
      "Iteration 100, loss = 0.47742287\n",
      "Validation score: 0.784399\n",
      "Iteration 101, loss = 0.47665119\n",
      "Validation score: 0.788974\n",
      "Iteration 102, loss = 0.47604655\n",
      "Validation score: 0.793549\n",
      "Iteration 103, loss = 0.47636849\n",
      "Validation score: 0.790575\n",
      "Iteration 104, loss = 0.47546946\n",
      "Validation score: 0.790232\n",
      "Iteration 105, loss = 0.47541589\n",
      "Validation score: 0.786229\n",
      "Iteration 106, loss = 0.47587760\n",
      "Validation score: 0.794350\n",
      "Iteration 107, loss = 0.47474844\n",
      "Validation score: 0.790347\n",
      "Iteration 108, loss = 0.47546113\n",
      "Validation score: 0.793549\n",
      "Iteration 109, loss = 0.47579849\n",
      "Validation score: 0.785771\n",
      "Iteration 110, loss = 0.47476628\n",
      "Validation score: 0.794350\n",
      "Iteration 111, loss = 0.47449508\n",
      "Validation score: 0.786572\n",
      "Iteration 112, loss = 0.47489393\n",
      "Validation score: 0.789889\n",
      "Iteration 113, loss = 0.47428784\n",
      "Validation score: 0.792291\n",
      "Iteration 114, loss = 0.47375083\n",
      "Validation score: 0.787144\n",
      "Iteration 115, loss = 0.47326537\n",
      "Validation score: 0.794235\n",
      "Iteration 116, loss = 0.47358544\n",
      "Validation score: 0.791719\n",
      "Iteration 117, loss = 0.47351090\n",
      "Validation score: 0.790918\n",
      "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
      "count  1\n",
      "Fitting:  2\n",
      "Iteration 1, loss = 0.69231021\n",
      "Validation score: 0.613977\n",
      "Iteration 2, loss = 0.69025337\n",
      "Validation score: 0.526135\n",
      "Iteration 3, loss = 0.68787733\n",
      "Validation score: 0.640398\n",
      "Iteration 4, loss = 0.68509423\n",
      "Validation score: 0.594762\n",
      "Iteration 5, loss = 0.68147242\n",
      "Validation score: 0.623127\n",
      "Iteration 6, loss = 0.67634542\n",
      "Validation score: 0.591673\n",
      "Iteration 7, loss = 0.66980252\n",
      "Validation score: 0.670250\n",
      "Iteration 8, loss = 0.66221377\n",
      "Validation score: 0.653895\n",
      "Iteration 9, loss = 0.65388351\n",
      "Validation score: 0.664074\n",
      "Iteration 10, loss = 0.64514170\n",
      "Validation score: 0.703191\n",
      "Iteration 11, loss = 0.63600615\n",
      "Validation score: 0.700789\n",
      "Iteration 12, loss = 0.62769362\n",
      "Validation score: 0.707995\n",
      "Iteration 13, loss = 0.61934129\n",
      "Validation score: 0.719890\n",
      "Iteration 14, loss = 0.61133428\n",
      "Validation score: 0.721148\n",
      "Iteration 15, loss = 0.60383755\n",
      "Validation score: 0.728240\n",
      "Iteration 16, loss = 0.59707815\n",
      "Validation score: 0.723207\n",
      "Iteration 17, loss = 0.59086505\n",
      "Validation score: 0.736246\n",
      "Iteration 18, loss = 0.58371643\n",
      "Validation score: 0.740249\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 19, loss = 0.57842479\n",
      "Validation score: 0.738762\n",
      "Iteration 20, loss = 0.57277527\n",
      "Validation score: 0.745854\n",
      "Iteration 21, loss = 0.56713321\n",
      "Validation score: 0.751687\n",
      "Iteration 22, loss = 0.56232225\n",
      "Validation score: 0.752259\n",
      "Iteration 23, loss = 0.55823932\n",
      "Validation score: 0.756720\n",
      "Iteration 24, loss = 0.55361263\n",
      "Validation score: 0.748141\n",
      "Iteration 25, loss = 0.55029404\n",
      "Validation score: 0.757406\n",
      "Iteration 26, loss = 0.54603307\n",
      "Validation score: 0.761638\n",
      "Iteration 27, loss = 0.54259737\n",
      "Validation score: 0.764383\n",
      "Iteration 28, loss = 0.53966785\n",
      "Validation score: 0.761752\n",
      "Iteration 29, loss = 0.53719750\n",
      "Validation score: 0.758321\n",
      "Iteration 30, loss = 0.53417311\n",
      "Validation score: 0.767014\n",
      "Iteration 31, loss = 0.53091961\n",
      "Validation score: 0.767929\n",
      "Iteration 32, loss = 0.52888568\n",
      "Validation score: 0.768844\n",
      "Iteration 33, loss = 0.52687219\n",
      "Validation score: 0.769873\n",
      "Iteration 34, loss = 0.52431832\n",
      "Validation score: 0.766785\n",
      "Iteration 35, loss = 0.52265185\n",
      "Validation score: 0.768729\n",
      "Iteration 36, loss = 0.52069010\n",
      "Validation score: 0.769987\n",
      "Iteration 37, loss = 0.51930600\n",
      "Validation score: 0.771131\n",
      "Iteration 38, loss = 0.51799490\n",
      "Validation score: 0.765527\n",
      "Iteration 39, loss = 0.51572110\n",
      "Validation score: 0.775363\n",
      "Iteration 40, loss = 0.51404402\n",
      "Validation score: 0.771360\n",
      "Iteration 41, loss = 0.51280131\n",
      "Validation score: 0.773304\n",
      "Iteration 42, loss = 0.51168243\n",
      "Validation score: 0.773876\n",
      "Iteration 43, loss = 0.51028381\n",
      "Validation score: 0.776393\n",
      "Iteration 44, loss = 0.50951161\n",
      "Validation score: 0.776278\n",
      "Iteration 45, loss = 0.50847252\n",
      "Validation score: 0.779138\n",
      "Iteration 46, loss = 0.50627183\n",
      "Validation score: 0.775706\n",
      "Iteration 47, loss = 0.50569501\n",
      "Validation score: 0.776393\n",
      "Iteration 48, loss = 0.50420838\n",
      "Validation score: 0.778794\n",
      "Iteration 49, loss = 0.50416448\n",
      "Validation score: 0.780510\n",
      "Iteration 50, loss = 0.50263571\n",
      "Validation score: 0.782226\n",
      "Iteration 51, loss = 0.50147388\n",
      "Validation score: 0.780968\n",
      "Iteration 52, loss = 0.50021738\n",
      "Validation score: 0.783370\n",
      "Iteration 53, loss = 0.49944965\n",
      "Validation score: 0.781997\n",
      "Iteration 54, loss = 0.49848907\n",
      "Validation score: 0.783370\n",
      "Iteration 55, loss = 0.49850081\n",
      "Validation score: 0.784285\n",
      "Iteration 56, loss = 0.49801401\n",
      "Validation score: 0.783827\n",
      "Iteration 57, loss = 0.49688603\n",
      "Validation score: 0.784399\n",
      "Iteration 58, loss = 0.49600272\n",
      "Validation score: 0.783484\n",
      "Iteration 59, loss = 0.49545760\n",
      "Validation score: 0.784856\n",
      "Iteration 60, loss = 0.49472917\n",
      "Validation score: 0.785200\n",
      "Iteration 61, loss = 0.49360685\n",
      "Validation score: 0.785314\n",
      "Iteration 62, loss = 0.49387573\n",
      "Validation score: 0.784971\n",
      "Iteration 63, loss = 0.49242052\n",
      "Validation score: 0.786915\n",
      "Iteration 64, loss = 0.49131539\n",
      "Validation score: 0.784742\n",
      "Iteration 65, loss = 0.49127991\n",
      "Validation score: 0.786229\n",
      "Iteration 66, loss = 0.49049748\n",
      "Validation score: 0.772618\n",
      "Iteration 67, loss = 0.49113637\n",
      "Validation score: 0.785314\n",
      "Iteration 68, loss = 0.48916025\n",
      "Validation score: 0.778794\n",
      "Iteration 69, loss = 0.48897867\n",
      "Validation score: 0.788288\n",
      "Iteration 70, loss = 0.48884280\n",
      "Validation score: 0.779023\n",
      "Iteration 71, loss = 0.48790712\n",
      "Validation score: 0.786115\n",
      "Iteration 72, loss = 0.48754926\n",
      "Validation score: 0.785771\n",
      "Iteration 73, loss = 0.48685573\n",
      "Validation score: 0.787144\n",
      "Iteration 74, loss = 0.48693547\n",
      "Validation score: 0.788517\n",
      "Iteration 75, loss = 0.48572198\n",
      "Validation score: 0.787716\n",
      "Iteration 76, loss = 0.48631027\n",
      "Validation score: 0.782683\n",
      "Iteration 77, loss = 0.48480783\n",
      "Validation score: 0.788745\n",
      "Iteration 78, loss = 0.48516431\n",
      "Validation score: 0.788402\n",
      "Iteration 79, loss = 0.48440158\n",
      "Validation score: 0.787030\n",
      "Iteration 80, loss = 0.48327920\n",
      "Validation score: 0.788860\n",
      "Iteration 81, loss = 0.48306714\n",
      "Validation score: 0.789317\n",
      "Iteration 82, loss = 0.48289228\n",
      "Validation score: 0.789889\n",
      "Iteration 83, loss = 0.48198166\n",
      "Validation score: 0.787602\n",
      "Iteration 84, loss = 0.48166243\n",
      "Validation score: 0.786115\n",
      "Iteration 85, loss = 0.48203680\n",
      "Validation score: 0.788288\n",
      "Iteration 86, loss = 0.48204751\n",
      "Validation score: 0.787144\n",
      "Iteration 87, loss = 0.48112997\n",
      "Validation score: 0.791262\n",
      "Iteration 88, loss = 0.48014508\n",
      "Validation score: 0.787030\n",
      "Iteration 89, loss = 0.48077195\n",
      "Validation score: 0.787487\n",
      "Iteration 90, loss = 0.47963505\n",
      "Validation score: 0.787258\n",
      "Iteration 91, loss = 0.47992526\n",
      "Validation score: 0.778566\n",
      "Iteration 92, loss = 0.47879894\n",
      "Validation score: 0.789546\n",
      "Iteration 93, loss = 0.47876577\n",
      "Validation score: 0.787373\n",
      "Iteration 94, loss = 0.47862943\n",
      "Validation score: 0.787144\n",
      "Iteration 95, loss = 0.47812547\n",
      "Validation score: 0.788974\n",
      "Iteration 96, loss = 0.47818171\n",
      "Validation score: 0.788402\n",
      "Iteration 97, loss = 0.47768495\n",
      "Validation score: 0.787830\n",
      "Iteration 98, loss = 0.47719166\n",
      "Validation score: 0.790232\n",
      "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
      "count  2\n",
      "Fitting:  3\n",
      "Iteration 1, loss = 0.69250299\n",
      "Validation score: 0.514240\n",
      "Iteration 2, loss = 0.69029884\n",
      "Validation score: 0.629761\n",
      "Iteration 3, loss = 0.68790471\n",
      "Validation score: 0.638110\n",
      "Iteration 4, loss = 0.68547851\n",
      "Validation score: 0.642571\n",
      "Iteration 5, loss = 0.68187510\n",
      "Validation score: 0.651607\n",
      "Iteration 6, loss = 0.67704571\n",
      "Validation score: 0.671737\n",
      "Iteration 7, loss = 0.67109704\n",
      "Validation score: 0.686149\n",
      "Iteration 8, loss = 0.66375781\n",
      "Validation score: 0.692097\n",
      "Iteration 9, loss = 0.65530141\n",
      "Validation score: 0.694727\n",
      "Iteration 10, loss = 0.64639994\n",
      "Validation score: 0.702619\n",
      "Iteration 11, loss = 0.63750570\n",
      "Validation score: 0.667505\n",
      "Iteration 12, loss = 0.62834260\n",
      "Validation score: 0.690610\n",
      "Iteration 13, loss = 0.61979992\n",
      "Validation score: 0.718861\n",
      "Iteration 14, loss = 0.61149704\n",
      "Validation score: 0.723550\n",
      "Iteration 15, loss = 0.60362513\n",
      "Validation score: 0.730756\n",
      "Iteration 16, loss = 0.59680433\n",
      "Validation score: 0.733158\n",
      "Iteration 17, loss = 0.58926961\n",
      "Validation score: 0.735102\n",
      "Iteration 18, loss = 0.58295623\n",
      "Validation score: 0.745739\n",
      "Iteration 19, loss = 0.57712337\n",
      "Validation score: 0.746654\n",
      "Iteration 20, loss = 0.57106078\n",
      "Validation score: 0.742537\n",
      "Iteration 21, loss = 0.56583616\n",
      "Validation score: 0.752030\n",
      "Iteration 22, loss = 0.56072194\n",
      "Validation score: 0.757978\n",
      "Iteration 23, loss = 0.55661865\n",
      "Validation score: 0.761981\n",
      "Iteration 24, loss = 0.55214086\n",
      "Validation score: 0.763125\n",
      "Iteration 25, loss = 0.54769732\n",
      "Validation score: 0.765412\n",
      "Iteration 26, loss = 0.54391896\n",
      "Validation score: 0.757063\n",
      "Iteration 27, loss = 0.54121050\n",
      "Validation score: 0.765527\n",
      "Iteration 28, loss = 0.53756001\n",
      "Validation score: 0.769759\n",
      "Iteration 29, loss = 0.53491864\n",
      "Validation score: 0.766442\n",
      "Iteration 30, loss = 0.53219039\n",
      "Validation score: 0.769987\n",
      "Iteration 31, loss = 0.52963069\n",
      "Validation score: 0.771932\n",
      "Iteration 32, loss = 0.52677852\n",
      "Validation score: 0.771703\n",
      "Iteration 33, loss = 0.52478101\n",
      "Validation score: 0.773762\n",
      "Iteration 34, loss = 0.52239247\n",
      "Validation score: 0.773304\n",
      "Iteration 35, loss = 0.52104959\n",
      "Validation score: 0.775249\n",
      "Iteration 36, loss = 0.51901575\n",
      "Validation score: 0.773647\n",
      "Iteration 37, loss = 0.51713702\n",
      "Validation score: 0.775020\n",
      "Iteration 38, loss = 0.51540865\n",
      "Validation score: 0.771932\n",
      "Iteration 39, loss = 0.51410450\n",
      "Validation score: 0.777079\n",
      "Iteration 40, loss = 0.51224566\n",
      "Validation score: 0.778680\n",
      "Iteration 41, loss = 0.51177659\n",
      "Validation score: 0.778108\n",
      "Iteration 42, loss = 0.50976589\n",
      "Validation score: 0.778337\n",
      "Iteration 43, loss = 0.50822781\n",
      "Validation score: 0.778909\n",
      "Iteration 44, loss = 0.50765252\n",
      "Validation score: 0.780167\n",
      "Iteration 45, loss = 0.50646678\n",
      "Validation score: 0.780053\n",
      "Iteration 46, loss = 0.50591940\n",
      "Validation score: 0.780396\n",
      "Iteration 47, loss = 0.50430382\n",
      "Validation score: 0.781311\n",
      "Iteration 48, loss = 0.50350273\n",
      "Validation score: 0.780510\n",
      "Iteration 49, loss = 0.50239572\n",
      "Validation score: 0.767471\n",
      "Iteration 50, loss = 0.50205975\n",
      "Validation score: 0.782340\n",
      "Iteration 51, loss = 0.50050166\n",
      "Validation score: 0.782569\n",
      "Iteration 52, loss = 0.49943880\n",
      "Validation score: 0.782683\n",
      "Iteration 53, loss = 0.49868063\n",
      "Validation score: 0.781082\n",
      "Iteration 54, loss = 0.49856322\n",
      "Validation score: 0.783941\n",
      "Iteration 55, loss = 0.49775562\n",
      "Validation score: 0.784513\n",
      "Iteration 56, loss = 0.49590376\n",
      "Validation score: 0.784399\n",
      "Iteration 57, loss = 0.49551067\n",
      "Validation score: 0.784399\n",
      "Iteration 58, loss = 0.49492749\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation score: 0.784856\n",
      "Iteration 59, loss = 0.49396801\n",
      "Validation score: 0.786801\n",
      "Iteration 60, loss = 0.49381789\n",
      "Validation score: 0.785886\n",
      "Iteration 61, loss = 0.49308003\n",
      "Validation score: 0.779938\n",
      "Iteration 62, loss = 0.49212865\n",
      "Validation score: 0.785543\n",
      "Iteration 63, loss = 0.49282856\n",
      "Validation score: 0.786572\n",
      "Iteration 64, loss = 0.49087048\n",
      "Validation score: 0.788402\n",
      "Iteration 65, loss = 0.49065242\n",
      "Validation score: 0.786686\n",
      "Iteration 66, loss = 0.49022405\n",
      "Validation score: 0.787258\n",
      "Iteration 67, loss = 0.48950704\n",
      "Validation score: 0.785428\n",
      "Iteration 68, loss = 0.48872699\n",
      "Validation score: 0.788059\n",
      "Iteration 69, loss = 0.48874785\n",
      "Validation score: 0.789317\n",
      "Iteration 70, loss = 0.48761253\n",
      "Validation score: 0.789889\n",
      "Iteration 71, loss = 0.48765170\n",
      "Validation score: 0.789088\n",
      "Iteration 72, loss = 0.48771012\n",
      "Validation score: 0.784056\n",
      "Iteration 73, loss = 0.48610274\n",
      "Validation score: 0.788631\n",
      "Iteration 74, loss = 0.48620546\n",
      "Validation score: 0.785314\n",
      "Iteration 75, loss = 0.48630276\n",
      "Validation score: 0.788288\n",
      "Iteration 76, loss = 0.48561199\n",
      "Validation score: 0.791147\n",
      "Iteration 77, loss = 0.48581165\n",
      "Validation score: 0.790918\n",
      "Iteration 78, loss = 0.48459745\n",
      "Validation score: 0.791262\n",
      "Iteration 79, loss = 0.48410099\n",
      "Validation score: 0.787144\n",
      "Iteration 80, loss = 0.48338837\n",
      "Validation score: 0.790690\n",
      "Iteration 81, loss = 0.48274129\n",
      "Validation score: 0.786915\n",
      "Iteration 82, loss = 0.48367382\n",
      "Validation score: 0.788059\n",
      "Iteration 83, loss = 0.48380123\n",
      "Validation score: 0.790347\n",
      "Iteration 84, loss = 0.48222662\n",
      "Validation score: 0.788173\n",
      "Iteration 85, loss = 0.48191859\n",
      "Validation score: 0.790918\n",
      "Iteration 86, loss = 0.48154496\n",
      "Validation score: 0.791262\n",
      "Iteration 87, loss = 0.48100491\n",
      "Validation score: 0.791376\n",
      "Iteration 88, loss = 0.48127040\n",
      "Validation score: 0.791147\n",
      "Iteration 89, loss = 0.48060257\n",
      "Validation score: 0.791605\n",
      "Iteration 90, loss = 0.47998378\n",
      "Validation score: 0.791033\n",
      "Iteration 91, loss = 0.48011791\n",
      "Validation score: 0.792748\n",
      "Iteration 92, loss = 0.48037749\n",
      "Validation score: 0.791948\n",
      "Iteration 93, loss = 0.47964025\n",
      "Validation score: 0.791833\n",
      "Iteration 94, loss = 0.47901825\n",
      "Validation score: 0.790690\n",
      "Iteration 95, loss = 0.47905352\n",
      "Validation score: 0.786915\n",
      "Iteration 96, loss = 0.47847653\n",
      "Validation score: 0.791490\n",
      "Iteration 97, loss = 0.47740648\n",
      "Validation score: 0.792520\n",
      "Iteration 98, loss = 0.47873101\n",
      "Validation score: 0.786572\n",
      "Iteration 99, loss = 0.47777760\n",
      "Validation score: 0.773533\n",
      "Iteration 100, loss = 0.47738816\n",
      "Validation score: 0.792977\n",
      "Iteration 101, loss = 0.47685482\n",
      "Validation score: 0.792977\n",
      "Iteration 102, loss = 0.47764163\n",
      "Validation score: 0.792977\n",
      "Iteration 103, loss = 0.47631212\n",
      "Validation score: 0.787258\n",
      "Iteration 104, loss = 0.47700580\n",
      "Validation score: 0.792634\n",
      "Iteration 105, loss = 0.47622973\n",
      "Validation score: 0.792405\n",
      "Iteration 106, loss = 0.47649879\n",
      "Validation score: 0.793892\n",
      "Iteration 107, loss = 0.47581445\n",
      "Validation score: 0.792977\n",
      "Iteration 108, loss = 0.47540101\n",
      "Validation score: 0.789660\n",
      "Iteration 109, loss = 0.47513929\n",
      "Validation score: 0.794007\n",
      "Iteration 110, loss = 0.47448261\n",
      "Validation score: 0.791605\n",
      "Iteration 111, loss = 0.47494131\n",
      "Validation score: 0.791719\n",
      "Iteration 112, loss = 0.47489880\n",
      "Validation score: 0.793092\n",
      "Iteration 113, loss = 0.47458202\n",
      "Validation score: 0.791719\n",
      "Iteration 114, loss = 0.47381070\n",
      "Validation score: 0.792977\n",
      "Iteration 115, loss = 0.47376639\n",
      "Validation score: 0.791948\n",
      "Iteration 116, loss = 0.47358800\n",
      "Validation score: 0.792977\n",
      "Iteration 117, loss = 0.47327751\n",
      "Validation score: 0.787030\n",
      "Iteration 118, loss = 0.47369109\n",
      "Validation score: 0.791948\n",
      "Iteration 119, loss = 0.47253105\n",
      "Validation score: 0.794579\n",
      "Iteration 120, loss = 0.47323883\n",
      "Validation score: 0.792062\n",
      "Iteration 121, loss = 0.47291584\n",
      "Validation score: 0.794464\n",
      "Iteration 122, loss = 0.47306496\n",
      "Validation score: 0.793664\n",
      "Iteration 123, loss = 0.47275580\n",
      "Validation score: 0.784628\n",
      "Iteration 124, loss = 0.47219759\n",
      "Validation score: 0.795951\n",
      "Iteration 125, loss = 0.47115810\n",
      "Validation score: 0.788974\n",
      "Iteration 126, loss = 0.47146255\n",
      "Validation score: 0.785771\n",
      "Iteration 127, loss = 0.47223792\n",
      "Validation score: 0.786000\n",
      "Iteration 128, loss = 0.47204276\n",
      "Validation score: 0.793549\n",
      "Iteration 129, loss = 0.47173006\n",
      "Validation score: 0.794579\n",
      "Iteration 130, loss = 0.47070402\n",
      "Validation score: 0.791262\n",
      "Iteration 131, loss = 0.47064303\n",
      "Validation score: 0.794007\n",
      "Iteration 132, loss = 0.47116399\n",
      "Validation score: 0.792863\n",
      "Iteration 133, loss = 0.47025517\n",
      "Validation score: 0.793778\n",
      "Iteration 134, loss = 0.47073948\n",
      "Validation score: 0.792977\n",
      "Iteration 135, loss = 0.46969627\n",
      "Validation score: 0.794235\n",
      "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
      "count  3\n",
      "Fitting:  4\n",
      "Iteration 1, loss = 0.69247974\n",
      "Validation score: 0.524305\n",
      "Iteration 2, loss = 0.69031563\n",
      "Validation score: 0.612147\n",
      "Iteration 3, loss = 0.68754011\n",
      "Validation score: 0.640169\n",
      "Iteration 4, loss = 0.68468667\n",
      "Validation score: 0.583781\n",
      "Iteration 5, loss = 0.68064467\n",
      "Validation score: 0.620611\n",
      "Iteration 6, loss = 0.67534626\n",
      "Validation score: 0.686378\n",
      "Iteration 7, loss = 0.66802826\n",
      "Validation score: 0.693126\n",
      "Iteration 8, loss = 0.65960350\n",
      "Validation score: 0.669679\n",
      "Iteration 9, loss = 0.64990035\n",
      "Validation score: 0.701704\n",
      "Iteration 10, loss = 0.64083859\n",
      "Validation score: 0.706394\n",
      "Iteration 11, loss = 0.63051770\n",
      "Validation score: 0.702276\n",
      "Iteration 12, loss = 0.62140336\n",
      "Validation score: 0.707995\n",
      "Iteration 13, loss = 0.61241010\n",
      "Validation score: 0.725838\n",
      "Iteration 14, loss = 0.60374465\n",
      "Validation score: 0.728125\n",
      "Iteration 15, loss = 0.59657833\n",
      "Validation score: 0.730299\n",
      "Iteration 16, loss = 0.58856668\n",
      "Validation score: 0.740936\n",
      "Iteration 17, loss = 0.58173776\n",
      "Validation score: 0.740707\n",
      "Iteration 18, loss = 0.57530987\n",
      "Validation score: 0.751230\n",
      "Iteration 19, loss = 0.56967804\n",
      "Validation score: 0.750543\n",
      "Iteration 20, loss = 0.56337777\n",
      "Validation score: 0.752831\n",
      "Iteration 21, loss = 0.55833367\n",
      "Validation score: 0.748485\n",
      "Iteration 22, loss = 0.55323458\n",
      "Validation score: 0.757177\n",
      "Iteration 23, loss = 0.54885589\n",
      "Validation score: 0.767128\n",
      "Iteration 24, loss = 0.54478353\n",
      "Validation score: 0.767814\n",
      "Iteration 25, loss = 0.54089380\n",
      "Validation score: 0.763010\n",
      "Iteration 26, loss = 0.53715448\n",
      "Validation score: 0.769987\n",
      "Iteration 27, loss = 0.53403797\n",
      "Validation score: 0.774105\n",
      "Iteration 28, loss = 0.53191620\n",
      "Validation score: 0.773076\n",
      "Iteration 29, loss = 0.52916168\n",
      "Validation score: 0.774677\n",
      "Iteration 30, loss = 0.52573772\n",
      "Validation score: 0.775592\n",
      "Iteration 31, loss = 0.52403446\n",
      "Validation score: 0.773647\n",
      "Iteration 32, loss = 0.52121107\n",
      "Validation score: 0.778566\n",
      "Iteration 33, loss = 0.51966963\n",
      "Validation score: 0.775020\n",
      "Iteration 34, loss = 0.51730452\n",
      "Validation score: 0.773533\n",
      "Iteration 35, loss = 0.51589001\n",
      "Validation score: 0.780853\n",
      "Iteration 36, loss = 0.51403462\n",
      "Validation score: 0.782226\n",
      "Iteration 37, loss = 0.51217669\n",
      "Validation score: 0.783484\n",
      "Iteration 38, loss = 0.51107053\n",
      "Validation score: 0.783141\n",
      "Iteration 39, loss = 0.50953003\n",
      "Validation score: 0.780510\n",
      "Iteration 40, loss = 0.50758790\n",
      "Validation score: 0.783370\n",
      "Iteration 41, loss = 0.50721697\n",
      "Validation score: 0.784856\n",
      "Iteration 42, loss = 0.50561531\n",
      "Validation score: 0.786915\n",
      "Iteration 43, loss = 0.50465575\n",
      "Validation score: 0.785200\n",
      "Iteration 44, loss = 0.50329787\n",
      "Validation score: 0.788059\n",
      "Iteration 45, loss = 0.50202674\n",
      "Validation score: 0.783941\n",
      "Iteration 46, loss = 0.50146454\n",
      "Validation score: 0.788631\n",
      "Iteration 47, loss = 0.50041308\n",
      "Validation score: 0.787945\n",
      "Iteration 48, loss = 0.49912922\n",
      "Validation score: 0.789088\n",
      "Iteration 49, loss = 0.49903955\n",
      "Validation score: 0.777308\n",
      "Iteration 50, loss = 0.49768819\n",
      "Validation score: 0.781654\n",
      "Iteration 51, loss = 0.49724530\n",
      "Validation score: 0.787373\n",
      "Iteration 52, loss = 0.49548477\n",
      "Validation score: 0.788517\n",
      "Iteration 53, loss = 0.49520720\n",
      "Validation score: 0.789317\n",
      "Iteration 54, loss = 0.49379423\n",
      "Validation score: 0.789203\n",
      "Iteration 55, loss = 0.49292786\n",
      "Validation score: 0.788860\n",
      "Iteration 56, loss = 0.49372057\n",
      "Validation score: 0.790804\n",
      "Iteration 57, loss = 0.49204280\n",
      "Validation score: 0.784513\n",
      "Iteration 58, loss = 0.49091950\n",
      "Validation score: 0.790118\n",
      "Iteration 59, loss = 0.49039810\n",
      "Validation score: 0.788288\n",
      "Iteration 60, loss = 0.48950013\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation score: 0.788974\n",
      "Iteration 61, loss = 0.48975575\n",
      "Validation score: 0.791033\n",
      "Iteration 62, loss = 0.48819170\n",
      "Validation score: 0.792177\n",
      "Iteration 63, loss = 0.48806723\n",
      "Validation score: 0.786572\n",
      "Iteration 64, loss = 0.48689949\n",
      "Validation score: 0.790347\n",
      "Iteration 65, loss = 0.48717800\n",
      "Validation score: 0.791833\n",
      "Iteration 66, loss = 0.48756891\n",
      "Validation score: 0.780281\n",
      "Iteration 67, loss = 0.48556979\n",
      "Validation score: 0.790575\n",
      "Iteration 68, loss = 0.48513036\n",
      "Validation score: 0.793664\n",
      "Iteration 69, loss = 0.48456289\n",
      "Validation score: 0.794235\n",
      "Iteration 70, loss = 0.48445201\n",
      "Validation score: 0.791948\n",
      "Iteration 71, loss = 0.48418221\n",
      "Validation score: 0.794007\n",
      "Iteration 72, loss = 0.48294217\n",
      "Validation score: 0.791605\n",
      "Iteration 73, loss = 0.48324460\n",
      "Validation score: 0.791833\n",
      "Iteration 74, loss = 0.48312639\n",
      "Validation score: 0.795150\n",
      "Iteration 75, loss = 0.48184037\n",
      "Validation score: 0.796065\n",
      "Iteration 76, loss = 0.48179300\n",
      "Validation score: 0.794922\n",
      "Iteration 77, loss = 0.48103294\n",
      "Validation score: 0.793320\n",
      "Iteration 78, loss = 0.48032385\n",
      "Validation score: 0.794235\n",
      "Iteration 79, loss = 0.48016080\n",
      "Validation score: 0.789660\n",
      "Iteration 80, loss = 0.48066001\n",
      "Validation score: 0.795951\n",
      "Iteration 81, loss = 0.47977463\n",
      "Validation score: 0.787602\n",
      "Iteration 82, loss = 0.47966229\n",
      "Validation score: 0.795150\n",
      "Iteration 83, loss = 0.47930675\n",
      "Validation score: 0.792634\n",
      "Iteration 84, loss = 0.47807698\n",
      "Validation score: 0.793206\n",
      "Iteration 85, loss = 0.47818626\n",
      "Validation score: 0.796065\n",
      "Iteration 86, loss = 0.47745373\n",
      "Validation score: 0.791719\n",
      "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
      "count  4\n",
      "Fitting:  5\n",
      "Iteration 1, loss = 0.69201811\n",
      "Validation score: 0.598193\n",
      "Iteration 2, loss = 0.68922643\n",
      "Validation score: 0.523047\n",
      "Iteration 3, loss = 0.68640791\n",
      "Validation score: 0.603683\n",
      "Iteration 4, loss = 0.68251195\n",
      "Validation score: 0.590987\n",
      "Iteration 5, loss = 0.67764607\n",
      "Validation score: 0.675741\n",
      "Iteration 6, loss = 0.67132098\n",
      "Validation score: 0.653323\n",
      "Iteration 7, loss = 0.66334430\n",
      "Validation score: 0.678943\n",
      "Iteration 8, loss = 0.65427685\n",
      "Validation score: 0.699417\n",
      "Iteration 9, loss = 0.64481485\n",
      "Validation score: 0.698159\n",
      "Iteration 10, loss = 0.63544884\n",
      "Validation score: 0.710168\n",
      "Iteration 11, loss = 0.62650198\n",
      "Validation score: 0.698502\n",
      "Iteration 12, loss = 0.61718100\n",
      "Validation score: 0.712684\n",
      "Iteration 13, loss = 0.60876404\n",
      "Validation score: 0.723779\n",
      "Iteration 14, loss = 0.60056551\n",
      "Validation score: 0.727325\n",
      "Iteration 15, loss = 0.59344134\n",
      "Validation score: 0.732014\n",
      "Iteration 16, loss = 0.58619025\n",
      "Validation score: 0.735789\n",
      "Iteration 17, loss = 0.57942376\n",
      "Validation score: 0.739677\n",
      "Iteration 18, loss = 0.57313825\n",
      "Validation score: 0.734988\n",
      "Iteration 19, loss = 0.56806898\n",
      "Validation score: 0.734645\n",
      "Iteration 20, loss = 0.56184873\n",
      "Validation score: 0.746654\n",
      "Iteration 21, loss = 0.55764662\n",
      "Validation score: 0.750543\n",
      "Iteration 22, loss = 0.55261097\n",
      "Validation score: 0.744024\n",
      "Iteration 23, loss = 0.54806026\n",
      "Validation score: 0.753060\n",
      "Iteration 24, loss = 0.54410648\n",
      "Validation score: 0.754775\n",
      "Iteration 25, loss = 0.54015403\n",
      "Validation score: 0.756948\n",
      "Iteration 26, loss = 0.53738346\n",
      "Validation score: 0.756491\n",
      "Iteration 27, loss = 0.53357664\n",
      "Validation score: 0.759922\n",
      "Iteration 28, loss = 0.53076124\n",
      "Validation score: 0.760723\n",
      "Iteration 29, loss = 0.52815781\n",
      "Validation score: 0.761867\n",
      "Iteration 30, loss = 0.52575905\n",
      "Validation score: 0.752145\n",
      "Iteration 31, loss = 0.52363317\n",
      "Validation score: 0.760151\n",
      "Iteration 32, loss = 0.52082926\n",
      "Validation score: 0.765755\n",
      "Iteration 33, loss = 0.51866259\n",
      "Validation score: 0.757863\n",
      "Iteration 34, loss = 0.51708262\n",
      "Validation score: 0.766899\n",
      "Iteration 35, loss = 0.51495606\n",
      "Validation score: 0.768501\n",
      "Iteration 36, loss = 0.51405232\n",
      "Validation score: 0.768615\n",
      "Iteration 37, loss = 0.51198662\n",
      "Validation score: 0.768729\n",
      "Iteration 38, loss = 0.51107376\n",
      "Validation score: 0.770331\n",
      "Iteration 39, loss = 0.50888938\n",
      "Validation score: 0.769416\n",
      "Iteration 40, loss = 0.50743485\n",
      "Validation score: 0.772504\n",
      "Iteration 41, loss = 0.50650371\n",
      "Validation score: 0.770102\n",
      "Iteration 42, loss = 0.50511995\n",
      "Validation score: 0.772847\n",
      "Iteration 43, loss = 0.50360486\n",
      "Validation score: 0.773419\n",
      "Iteration 44, loss = 0.50321701\n",
      "Validation score: 0.773076\n",
      "Iteration 45, loss = 0.50224010\n",
      "Validation score: 0.772732\n",
      "Iteration 46, loss = 0.50074388\n",
      "Validation score: 0.770216\n",
      "Iteration 47, loss = 0.49985708\n",
      "Validation score: 0.775706\n",
      "Iteration 48, loss = 0.49903880\n",
      "Validation score: 0.775249\n",
      "Iteration 49, loss = 0.49806147\n",
      "Validation score: 0.773762\n",
      "Iteration 50, loss = 0.49682842\n",
      "Validation score: 0.775363\n",
      "Iteration 51, loss = 0.49573209\n",
      "Validation score: 0.777765\n",
      "Iteration 52, loss = 0.49578128\n",
      "Validation score: 0.775592\n",
      "Iteration 53, loss = 0.49382643\n",
      "Validation score: 0.776278\n",
      "Iteration 54, loss = 0.49418351\n",
      "Validation score: 0.777193\n",
      "Iteration 55, loss = 0.49269739\n",
      "Validation score: 0.779709\n",
      "Iteration 56, loss = 0.49276287\n",
      "Validation score: 0.778108\n",
      "Iteration 57, loss = 0.49152017\n",
      "Validation score: 0.777193\n",
      "Iteration 58, loss = 0.49116234\n",
      "Validation score: 0.775020\n",
      "Iteration 59, loss = 0.49065439\n",
      "Validation score: 0.777994\n",
      "Iteration 60, loss = 0.48873810\n",
      "Validation score: 0.778794\n",
      "Iteration 61, loss = 0.48894600\n",
      "Validation score: 0.779481\n",
      "Iteration 62, loss = 0.48872944\n",
      "Validation score: 0.771131\n",
      "Iteration 63, loss = 0.48808207\n",
      "Validation score: 0.778909\n",
      "Iteration 64, loss = 0.48691708\n",
      "Validation score: 0.780968\n",
      "Iteration 65, loss = 0.48709350\n",
      "Validation score: 0.778337\n",
      "Iteration 66, loss = 0.48619226\n",
      "Validation score: 0.775821\n",
      "Iteration 67, loss = 0.48571684\n",
      "Validation score: 0.778909\n",
      "Iteration 68, loss = 0.48479664\n",
      "Validation score: 0.781311\n",
      "Iteration 69, loss = 0.48412688\n",
      "Validation score: 0.781082\n",
      "Iteration 70, loss = 0.48361678\n",
      "Validation score: 0.777765\n",
      "Iteration 71, loss = 0.48354857\n",
      "Validation score: 0.781311\n",
      "Iteration 72, loss = 0.48283465\n",
      "Validation score: 0.781540\n",
      "Iteration 73, loss = 0.48306918\n",
      "Validation score: 0.774791\n",
      "Iteration 74, loss = 0.48265971\n",
      "Validation score: 0.782569\n",
      "Iteration 75, loss = 0.48146692\n",
      "Validation score: 0.781654\n",
      "Iteration 76, loss = 0.48110327\n",
      "Validation score: 0.783026\n",
      "Iteration 77, loss = 0.48034615\n",
      "Validation score: 0.782340\n",
      "Iteration 78, loss = 0.47988693\n",
      "Validation score: 0.781997\n",
      "Iteration 79, loss = 0.47978292\n",
      "Validation score: 0.782683\n",
      "Iteration 80, loss = 0.47914573\n",
      "Validation score: 0.782340\n",
      "Iteration 81, loss = 0.47986097\n",
      "Validation score: 0.783255\n",
      "Iteration 82, loss = 0.47908825\n",
      "Validation score: 0.779824\n",
      "Iteration 83, loss = 0.47878244\n",
      "Validation score: 0.781196\n",
      "Iteration 84, loss = 0.47836612\n",
      "Validation score: 0.783941\n",
      "Iteration 85, loss = 0.47724814\n",
      "Validation score: 0.783484\n",
      "Iteration 86, loss = 0.47757353\n",
      "Validation score: 0.781540\n",
      "Iteration 87, loss = 0.47692996\n",
      "Validation score: 0.784285\n",
      "Iteration 88, loss = 0.47655444\n",
      "Validation score: 0.784742\n",
      "Iteration 89, loss = 0.47705418\n",
      "Validation score: 0.782798\n",
      "Iteration 90, loss = 0.47607754\n",
      "Validation score: 0.782340\n",
      "Iteration 91, loss = 0.47537996\n",
      "Validation score: 0.781883\n",
      "Iteration 92, loss = 0.47559564\n",
      "Validation score: 0.784056\n",
      "Iteration 93, loss = 0.47614465\n",
      "Validation score: 0.781883\n",
      "Iteration 94, loss = 0.47530919\n",
      "Validation score: 0.784856\n",
      "Iteration 95, loss = 0.47511456\n",
      "Validation score: 0.783941\n",
      "Iteration 96, loss = 0.47400672\n",
      "Validation score: 0.779366\n",
      "Iteration 97, loss = 0.47471147\n",
      "Validation score: 0.785085\n",
      "Iteration 98, loss = 0.47414080\n",
      "Validation score: 0.784285\n",
      "Iteration 99, loss = 0.47353024\n",
      "Validation score: 0.776850\n",
      "Iteration 100, loss = 0.47378166\n",
      "Validation score: 0.784513\n",
      "Iteration 101, loss = 0.47295870\n",
      "Validation score: 0.779824\n",
      "Iteration 102, loss = 0.47251234\n",
      "Validation score: 0.784628\n",
      "Iteration 103, loss = 0.47267873\n",
      "Validation score: 0.782569\n",
      "Iteration 104, loss = 0.47205920\n",
      "Validation score: 0.785543\n",
      "Iteration 105, loss = 0.47188777\n",
      "Validation score: 0.784628\n",
      "Iteration 106, loss = 0.47235938\n",
      "Validation score: 0.783255\n",
      "Iteration 107, loss = 0.47099649\n",
      "Validation score: 0.785543\n",
      "Iteration 108, loss = 0.47161878\n",
      "Validation score: 0.785085\n",
      "Iteration 109, loss = 0.47091937\n",
      "Validation score: 0.784742\n",
      "Iteration 110, loss = 0.47138804\n",
      "Validation score: 0.785657\n",
      "Iteration 111, loss = 0.47144357\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation score: 0.781997\n",
      "Iteration 112, loss = 0.47162728\n",
      "Validation score: 0.781082\n",
      "Iteration 113, loss = 0.47146377\n",
      "Validation score: 0.786343\n",
      "Iteration 114, loss = 0.47038736\n",
      "Validation score: 0.785771\n",
      "Iteration 115, loss = 0.47006208\n",
      "Validation score: 0.787602\n",
      "Iteration 116, loss = 0.46995810\n",
      "Validation score: 0.781883\n",
      "Iteration 117, loss = 0.46940581\n",
      "Validation score: 0.782111\n",
      "Iteration 118, loss = 0.47011764\n",
      "Validation score: 0.787258\n",
      "Iteration 119, loss = 0.46989648\n",
      "Validation score: 0.786229\n",
      "Iteration 120, loss = 0.47079825\n",
      "Validation score: 0.780281\n",
      "Iteration 121, loss = 0.46837904\n",
      "Validation score: 0.787030\n",
      "Iteration 122, loss = 0.46823160\n",
      "Validation score: 0.783827\n",
      "Iteration 123, loss = 0.46950564\n",
      "Validation score: 0.783713\n",
      "Iteration 124, loss = 0.46872501\n",
      "Validation score: 0.786458\n",
      "Iteration 125, loss = 0.46785981\n",
      "Validation score: 0.784056\n",
      "Iteration 126, loss = 0.46744217\n",
      "Validation score: 0.787487\n",
      "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
      "count  5\n",
      "Name Neural Net . Avg Precision:  0.7910909078295487 . Avg Recall:  0.7907508346001981 . Avg F-1 Score:  0.7906968887054064\n",
      "Fitting:  1\n",
      "count  1\n",
      "Fitting:  2\n",
      "count  2\n",
      "Fitting:  3\n",
      "count  3\n",
      "Fitting:  4\n",
      "count  4\n",
      "Fitting:  5\n",
      "count  5\n",
      "Name Logistic Regression . Avg Precision:  0.6793874171438687 . Avg Recall:  0.6779093529246217 . Avg F-1 Score:  0.6772290998984843\n",
      "Fitting:  1\n",
      "count  1\n",
      "Fitting:  2\n",
      "count  2\n",
      "Fitting:  3\n",
      "count  3\n",
      "Fitting:  4\n",
      "count  4\n",
      "Fitting:  5\n",
      "count  5\n",
      "Name Linear SVC . Avg Precision:  0.5659663400288547 . Avg Recall:  0.5602731318815732 . Avg F-1 Score:  0.5505930487768769\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import precision_recall_fscore_support\n",
    "from sklearn.model_selection import KFold\n",
    "\n",
    "kfold = KFold(n_splits=5, shuffle=True)\n",
    "for name, clf in zip(names, classifiers):\n",
    "    precScores = []\n",
    "    recallScores = []\n",
    "    f1Scores = []\n",
    "    count = 1\n",
    "    for train_index, test_index in kfold.split(X, Y):\n",
    "        X_train, X_test = X[train_index], X[test_index]\n",
    "        y_train, y_test = Y[train_index], Y[test_index]\n",
    "        print('Fitting: ', count)\n",
    "        clf.fit(X_train, y_train)\n",
    "        print('count ', count)\n",
    "        y_pred = clf.predict(X_test)\n",
    "        prec, recall, fscore, _ = precision_recall_fscore_support(y_test, y_pred, average='weighted')\n",
    "        precScores.append(prec)\n",
    "        recallScores.append(recall)\n",
    "        f1Scores.append(fscore)\n",
    "        count += 1\n",
    "    print('Name', name,'. Avg Precision: ', average(precScores), '. Avg Recall: ', average(recallScores), '. Avg F-1 Score: ', average(f1Scores) )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}