XLNET_gossip.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from keras.layers import Dense, Flatten, Dropout, Input, BatchNormalization\n",
    "from keras.utils import np_utils\n",
    "from keras.models import Sequential, Model\n",
    "from keras import regularizers\n",
    "import torch\n",
    "from keras import optimizers\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from keras.layers.merge import concatenate\n",
    "from keras import initializers\n",
    "from keras.utils import to_categorical\n",
    "from keras.layers.advanced_activations import LeakyReLU, PReLU\n",
    "\n",
    "import os\n",
    "os.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\"\n",
    "import threading\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"2\"\n",
    "import pickle\n",
    "import gc\n",
    "\n",
    "import keras\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open('gossip/finalTrainEmbeddings.pkl', 'rb') as f:\n",
    "    trainEmbeddings = pickle.load(f)\n",
    "with open('gossip/finalTestEmbeddings.pkl', 'rb') as f:\n",
    "    testEmbeddings = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "with open('imageDataset/gossip/trainJson.json', 'r') as f:\n",
    "    trainData = json.load(f)\n",
    "with open('imageDataset/gossip/testJson.json', 'r') as f:\n",
    "    testData = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in trainEmbeddings:\n",
    "    if len(trainEmbeddings[i]) >=50:\n",
    "        trainEmbeddings[i] = trainEmbeddings[i][0:50]\n",
    "    else:\n",
    "        deficit = 50 - len(trainEmbeddings[i])\n",
    "        for j in range(deficit):\n",
    "            trainEmbeddings[i].append(np.zeros((1,768), dtype=np.float32))\n",
    "    temp = np.empty((50,768), dtype=np.float32)\n",
    "    for j in range(len(trainEmbeddings[i])):\n",
    "        temp[j][:] = trainEmbeddings[i][j]\n",
    "    trainEmbeddings[i] = temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('gossip/train_gossip.pickle', 'rb') as f:\n",
    "    train_vgg_poli = pickle.load(f)\n",
    "with open('gossip/test_gossip.pickle', 'rb') as f:\n",
    "    test_vgg_poli = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in testEmbeddings:\n",
    "    if len(testEmbeddings[i]) >=50:\n",
    "        testEmbeddings[i] = testEmbeddings[i][0:50]\n",
    "    else:\n",
    "        deficit = 50 - len(testEmbeddings[i])\n",
    "        for j in range(deficit):\n",
    "            testEmbeddings[i].append(np.zeros((1,768), dtype=np.float32))\n",
    "    temp = np.empty((50,768), dtype=np.float32)\n",
    "    for j in range(len(testEmbeddings[i])):\n",
    "        temp[j][:] = testEmbeddings[i][j]\n",
    "    testEmbeddings[i] = temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_text = [] # text embeddings\n",
    "train_label = [] # labels\n",
    "test_text = [] # text embeddings\n",
    "test_label = []\n",
    "train_image = [] # image embeddings\n",
    "test_image = []\n",
    "\n",
    "# Train Image IDs\n",
    "# Test Image IDs\n",
    "trainImageNames = [] # names of the images i.e name.jpg\n",
    "trainTextNames = []  # train articles\n",
    "testTextNames = []   # test articles\n",
    "testImageNames = []  # names of the images in the test folder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# note: finalTrainEmbeddings.pkl---> trainDataJson.json----> trainTextNames\n",
    "# trainDataJson.json is a dictionary: format---> key (article i.e title+text): [label, article_url, imageId]\n",
    "# In traindataJson.json , we have complete list of data with all the attributes and trainTextnames contains all the samples that have associated images with it. this is obtained by running the \"Match code\" given below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# match code\n",
    "# Check whether Image exists, then get corresponding Text Embeddings, and finally append to respective lists\n",
    "for i in train_vgg_poli:\n",
    "    for j in trainData:\n",
    "        if i.split('.jpg')[0] == trainData[j][-1]['Top_img']:\n",
    "            if j in trainEmbeddings:\n",
    "                trainImageNames.append(i)\n",
    "                trainTextNames.append(j)\n",
    "                train_text.append(trainEmbeddings[j])\n",
    "                train_image.append(train_vgg_poli[trainData[j][-1]['Top_img'] + '.jpg'])\n",
    "                train_label.append(trainData[j][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in test_vgg_poli:\n",
    "    for j in testData:\n",
    "        if i.split('.jpg')[0] == testData[j][-1]['Top_img']:\n",
    "            if j in testEmbeddings:\n",
    "                testImageNames.append(i)\n",
    "                testTextNames.append(j)\n",
    "                test_text.append(testEmbeddings[j])\n",
    "                test_image.append(test_vgg_poli[testData[j][-1]['Top_img'] + '.jpg'])\n",
    "                test_label.append(testData[j][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10010, 2830)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(trainTextNames), len(testTextNames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10010, 2830)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(trainImageNames), len(testImageNames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10010, 2830)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_label), len(test_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10259 2581\n"
     ]
    }
   ],
   "source": [
    "# to count total number of real and fake articles in train and test split\n",
    "realCount = 0\n",
    "fakeCount = 0\n",
    "for i in train_label:\n",
    "    if i == 1:\n",
    "        realCount += 1\n",
    "    elif i == 0:\n",
    "        fakeCount += 1\n",
    "\n",
    "for i in test_label:\n",
    "    if i == 1:\n",
    "        realCount += 1\n",
    "    elif i == 0:\n",
    "        fakeCount += 1\n",
    "print(realCount, fakeCount)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # saving updated final dataset in the csv files for future reference\n",
    "# import pandas as pd\n",
    "# df=pd.DataFrame()\n",
    "# # df['train_articles']=trainTextNames\n",
    "# # df.to_csv('gossip_train_articles.csv', index=False)\n",
    "\n",
    "# # df['test_articles']=testTextNames\n",
    "# # df.to_csv('gossip_test_articles.csv', index=False)\n",
    "\n",
    "# df['train_label']=train_label\n",
    "# df.to_csv('gossip_train_label.csv', index=False)\n",
    "\n",
    "# # df['test_label']=test_label\n",
    "# # df.to_csv('gossip_test_label.csv', index=False)\n",
    "\n",
    "# # df['train_image']=trainImageNames\n",
    "# # df.to_csv('gossip_train_image.csv', index=False)\n",
    "\n",
    "# # df['test_image']=testImageNames\n",
    "# # df.to_csv('gossip_test_image.csv', index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_label = to_categorical(train_label)\n",
    "test_label = to_categorical(test_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 4096)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"train_text=[torch.Tensor.numpy(i.cpu()) for i in train_text]\n",
    "test_text=[torch.Tensor.numpy(i.cpu()) for i in test_text]\"\"\"\n",
    "train_image[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_text_matrix = np.ndarray(shape=(len(train_text), 50,768))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "counter = 0\n",
    "for i in train_text:\n",
    "    train_text_matrix[counter][:][:] = i\n",
    "    counter += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_text_matrix = np.ndarray(shape=(len(test_text), 50,768))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "counter = 0\n",
    "for i in test_text:\n",
    "    test_text_matrix[counter][:][:] = i\n",
    "    counter += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2830, 50, 768)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_text_matrix.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 4096)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_image[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_image_matrix = np.ndarray(shape=(len(train_image), 4096,1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "counter = 0\n",
    "for i in train_image:\n",
    "    train_image_matrix[counter][:][:] = i.reshape(4096,1)\n",
    "    counter += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_image_matrix = np.ndarray(shape=(len(test_image), 4096,1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "counter = 0\n",
    "for i in test_image:\n",
    "    test_image_matrix[counter][:][:] = i.reshape(4096,1)\n",
    "    counter += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10010"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_image_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_image_matrix = train_image_matrix.reshape(len(train_image_matrix),4096)\n",
    "test_image_matrix = test_image_matrix.reshape(len(test_image_matrix),4096)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### XLNET+dense+VGG"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_1\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 50, 768)      0                                            \n",
      "__________________________________________________________________________________________________\n",
      "flatten_1 (Flatten)             (None, 38400)        0           input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "input_2 (InputLayer)            (None, 4096)         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 1000)         38401000    flatten_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_4 (Dense)                 (None, 2000)         8194000     input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 500)          500500      dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_5 (Dense)                 (None, 1000)         2001000     dense_4[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 100)          50100       dense_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_6 (Dense)                 (None, 100)          100100      dense_5[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_1 (BatchNor (None, 100)          400         dense_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_2 (BatchNor (None, 100)          400         dense_6[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, 100)          0           batch_normalization_1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)             (None, 100)          0           batch_normalization_2[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 200)          0           dropout_1[0][0]                  \n",
      "                                                                 dropout_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_7 (Dense)                 (None, 200)          40200       concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_8 (Dense)                 (None, 100)          20100       dense_7[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_9 (Dense)                 (None, 50)           5050        dense_8[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dropout_3 (Dropout)             (None, 50)           0           dense_9[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_10 (Dense)                (None, 2)            102         dropout_3[0][0]                  \n",
      "==================================================================================================\n",
      "Total params: 49,312,952\n",
      "Trainable params: 49,312,552\n",
      "Non-trainable params: 400\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "input_text = Input(shape=(50,768))\n",
    "text_flat = Flatten()(input_text)\n",
    "dense_text = Dense(1000,activation='relu',kernel_regularizer=regularizers.l2(0.01), kernel_initializer=initializers.he_normal(seed=0))(text_flat)\n",
    "#dense_text = Dropout(0.4)(dense_text)\n",
    "dense_text = Dense(500,activation='relu',kernel_regularizer=regularizers.l2(0.01), kernel_initializer=initializers.he_normal(seed=0))(dense_text)\n",
    "#dense_text = Dropout(0.4)(dense_text)\n",
    "dense_text = Dense(100,activation='relu',kernel_regularizer=regularizers.l2(0.01), kernel_initializer=initializers.he_normal(seed=0))(dense_text)\n",
    "dense_text = BatchNormalization()(dense_text)\n",
    "dense_text_drop = Dropout(0.4)(dense_text)\n",
    "\n",
    "input_image = Input(shape=(4096,))\n",
    "dense_image = Dense(2000,activation='relu',kernel_regularizer=regularizers.l2(0.01), kernel_initializer=initializers.he_normal(seed=0))(input_image)\n",
    "#dense_image = Dropout(0.4)(dense_image)\n",
    "dense_image = Dense(1000, activation='relu',kernel_regularizer=regularizers.l2(0.01), kernel_initializer=initializers.he_normal(seed=0))(dense_image)\n",
    "#dense_image = Dropout(0.4)(dense_image)\n",
    "dense_image = Dense(100,activation='relu',kernel_regularizer=regularizers.l2(0.01), kernel_initializer=initializers.he_normal(seed=0))(dense_image)\n",
    "dense_image = BatchNormalization()(dense_image)\n",
    "dense_image_drop = Dropout(0.4)(dense_image)\n",
    "\n",
    "concat = concatenate([dense_text_drop,dense_image_drop])\n",
    "\n",
    "inter1_dense = Dense(200,activation='relu',kernel_regularizer=regularizers.l2(0.01), kernel_initializer=initializers.he_normal(seed=0))(concat)\n",
    "inter1_dense = Dense(100,activation='relu',kernel_regularizer=regularizers.l2(0.01), kernel_initializer=initializers.he_normal(seed=0))(inter1_dense)\n",
    "final_dense = Dense(50,activation='relu',kernel_regularizer=regularizers.l2(0.01), kernel_initializer=initializers.he_normal(seed=0))(inter1_dense)\n",
    "final_dropout = Dropout(0.4)(final_dense)\n",
    "output = Dense(2, activation='softmax')(final_dropout)\n",
    "\n",
    "model = Model(inputs=[input_text,input_image], outputs=output)\n",
    "adam = optimizers.Adam(lr=1e-4)\n",
    "#adagrad = optimizers.Adagrad(lr=1e-4)\n",
    "#adamax = optimizers.Adamax(lr=0.002, beta_1=0.9, beta_2=0.999)\n",
    "checkpoint = ModelCheckpoint(filepath='../checkpoints_gossip/dense_MM_model.hdf5', monitor='val_acc', verbose=1, save_best_only=True, mode='max')\n",
    "callbacks_list = [checkpoint]\n",
    "\n",
    "#sgd = optimizers.SGD(lr=1e-4, clipnorm=1.)\n",
    "model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy'])\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 10010 samples, validate on 2830 samples\n",
      "Epoch 1/100\n",
      "10010/10010 [==============================] - 15s 1ms/step - loss: 73.6526 - accuracy: 0.7553 - val_loss: 54.0207 - val_accuracy: 0.8131\n",
      "Epoch 2/100\n",
      "  160/10010 [..............................] - ETA: 11s - loss: 53.8265 - accuracy: 0.7937"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/shivangi/.conda/envs/BERT/lib/python3.7/site-packages/keras/callbacks/callbacks.py:707: RuntimeWarning: Can save best model only with val_acc available, skipping.\n",
      "  'skipping.' % (self.monitor), RuntimeWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10010/10010 [==============================] - 13s 1ms/step - loss: 42.0623 - accuracy: 0.7956 - val_loss: 32.1249 - val_accuracy: 0.8134\n",
      "Epoch 3/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 25.4229 - accuracy: 0.8054 - val_loss: 19.8380 - val_accuracy: 0.8265\n",
      "Epoch 4/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 15.9667 - accuracy: 0.8267 - val_loss: 12.7478 - val_accuracy: 0.8170\n",
      "Epoch 5/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 10.4579 - accuracy: 0.8392 - val_loss: 8.5801 - val_accuracy: 0.8269\n",
      "Epoch 6/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 7.1422 - accuracy: 0.8489 - val_loss: 5.9884 - val_accuracy: 0.8329\n",
      "Epoch 7/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 5.0724 - accuracy: 0.8554 - val_loss: 4.3518 - val_accuracy: 0.8484\n",
      "Epoch 8/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 3.7515 - accuracy: 0.8635 - val_loss: 3.3314 - val_accuracy: 0.8102\n",
      "Epoch 9/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 2.8367 - accuracy: 0.8745 - val_loss: 3.4741 - val_accuracy: 0.5283\n",
      "Epoch 10/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 2.1993 - accuracy: 0.8838 - val_loss: 2.0726 - val_accuracy: 0.8307\n",
      "Epoch 11/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 1.7332 - accuracy: 0.8935 - val_loss: 1.9399 - val_accuracy: 0.8219\n",
      "Epoch 12/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 1.3704 - accuracy: 0.9127 - val_loss: 1.5389 - val_accuracy: 0.7360\n",
      "Epoch 13/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 1.0835 - accuracy: 0.9341 - val_loss: 1.2996 - val_accuracy: 0.7905\n",
      "Epoch 14/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.8600 - accuracy: 0.9500 - val_loss: 1.2086 - val_accuracy: 0.8364\n",
      "Epoch 15/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.6938 - accuracy: 0.9595 - val_loss: 1.0424 - val_accuracy: 0.8424\n",
      "Epoch 16/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.5770 - accuracy: 0.9648 - val_loss: 1.0241 - val_accuracy: 0.8389\n",
      "Epoch 17/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.4734 - accuracy: 0.9726 - val_loss: 1.0144 - val_accuracy: 0.8102\n",
      "Epoch 18/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.4100 - accuracy: 0.9747 - val_loss: 1.0220 - val_accuracy: 0.8456\n",
      "Epoch 19/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.3463 - accuracy: 0.9806 - val_loss: 1.0307 - val_accuracy: 0.7523\n",
      "Epoch 20/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.3097 - accuracy: 0.9808 - val_loss: 0.9566 - val_accuracy: 0.7746\n",
      "Epoch 21/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.2801 - accuracy: 0.9802 - val_loss: 0.8087 - val_accuracy: 0.8216\n",
      "Epoch 22/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.2523 - accuracy: 0.9822 - val_loss: 0.9710 - val_accuracy: 0.7661\n",
      "Epoch 23/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.2451 - accuracy: 0.9814 - val_loss: 0.8899 - val_accuracy: 0.8452\n",
      "Epoch 24/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.2140 - accuracy: 0.9874 - val_loss: 0.9354 - val_accuracy: 0.8357\n",
      "Epoch 25/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.2000 - accuracy: 0.9861 - val_loss: 0.7858 - val_accuracy: 0.8283\n",
      "Epoch 26/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.1984 - accuracy: 0.9861 - val_loss: 0.7955 - val_accuracy: 0.8307\n",
      "Epoch 27/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.1871 - accuracy: 0.9878 - val_loss: 0.7879 - val_accuracy: 0.8523\n",
      "Epoch 28/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.1731 - accuracy: 0.9896 - val_loss: 0.8147 - val_accuracy: 0.8237\n",
      "Epoch 29/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.1724 - accuracy: 0.9874 - val_loss: 0.8023 - val_accuracy: 0.8314\n",
      "Epoch 30/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.1587 - accuracy: 0.9899 - val_loss: 0.8722 - val_accuracy: 0.8314\n",
      "Epoch 31/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.1561 - accuracy: 0.9892 - val_loss: 0.8140 - val_accuracy: 0.8198\n",
      "Epoch 32/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.1561 - accuracy: 0.9885 - val_loss: 0.7685 - val_accuracy: 0.8438\n",
      "Epoch 33/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.1528 - accuracy: 0.9871 - val_loss: 0.8021 - val_accuracy: 0.8212\n",
      "Epoch 34/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.1422 - accuracy: 0.9899 - val_loss: 0.7973 - val_accuracy: 0.8233\n",
      "Epoch 35/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.1327 - accuracy: 0.9924 - val_loss: 0.8508 - val_accuracy: 0.8491\n",
      "Epoch 36/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.1417 - accuracy: 0.9874 - val_loss: 0.7329 - val_accuracy: 0.8322\n",
      "Epoch 37/100\n",
      "10010/10010 [==============================] - 12s 1ms/step - loss: 0.1338 - accuracy: 0.9896 - val_loss: 0.8187 - val_accuracy: 0.8346\n",
      "Epoch 38/100\n",
      "10010/10010 [==============================] - 12s 1ms/step - loss: 0.1252 - accuracy: 0.9916 - val_loss: 0.9679 - val_accuracy: 0.7530\n",
      "Epoch 39/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.1282 - accuracy: 0.9901 - val_loss: 0.8016 - val_accuracy: 0.8343\n",
      "Epoch 40/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.1279 - accuracy: 0.9894 - val_loss: 0.8764 - val_accuracy: 0.8498\n",
      "Epoch 41/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.1185 - accuracy: 0.9938 - val_loss: 0.7689 - val_accuracy: 0.8378\n",
      "Epoch 42/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.1157 - accuracy: 0.9922 - val_loss: 0.8312 - val_accuracy: 0.8428\n",
      "Epoch 43/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.1180 - accuracy: 0.9918 - val_loss: 0.7722 - val_accuracy: 0.8420\n",
      "Epoch 44/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.1149 - accuracy: 0.9919 - val_loss: 0.7982 - val_accuracy: 0.8505\n",
      "Epoch 45/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.1118 - accuracy: 0.9929 - val_loss: 0.8097 - val_accuracy: 0.8251\n",
      "Epoch 46/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.1142 - accuracy: 0.9917 - val_loss: 0.8291 - val_accuracy: 0.8512\n",
      "Epoch 47/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.1098 - accuracy: 0.9910 - val_loss: 0.8602 - val_accuracy: 0.8435\n",
      "Epoch 48/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.1102 - accuracy: 0.9901 - val_loss: 1.0778 - val_accuracy: 0.7470\n",
      "Epoch 49/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.1041 - accuracy: 0.9935 - val_loss: 0.8475 - val_accuracy: 0.8343\n",
      "Epoch 50/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.1146 - accuracy: 0.9894 - val_loss: 0.9175 - val_accuracy: 0.7869\n",
      "Epoch 51/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.1100 - accuracy: 0.9907 - val_loss: 0.8546 - val_accuracy: 0.8431\n",
      "Epoch 52/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.1094 - accuracy: 0.9899 - val_loss: 0.8820 - val_accuracy: 0.8145\n",
      "Epoch 53/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.1096 - accuracy: 0.9907 - val_loss: 0.8317 - val_accuracy: 0.8247\n",
      "Epoch 54/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.1031 - accuracy: 0.9919 - val_loss: 0.8340 - val_accuracy: 0.8272\n",
      "Epoch 55/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.1007 - accuracy: 0.9933 - val_loss: 0.8911 - val_accuracy: 0.7947\n",
      "Epoch 56/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.1005 - accuracy: 0.9923 - val_loss: 0.8475 - val_accuracy: 0.8336\n",
      "Epoch 57/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.0969 - accuracy: 0.9938 - val_loss: 0.8730 - val_accuracy: 0.8226\n",
      "Epoch 58/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.1047 - accuracy: 0.9913 - val_loss: 0.8248 - val_accuracy: 0.8283\n",
      "Epoch 59/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.0988 - accuracy: 0.9924 - val_loss: 0.8597 - val_accuracy: 0.8371\n",
      "Epoch 60/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.0926 - accuracy: 0.9941 - val_loss: 0.8608 - val_accuracy: 0.8375\n",
      "Epoch 61/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.0987 - accuracy: 0.9922 - val_loss: 0.8661 - val_accuracy: 0.8343\n",
      "Epoch 62/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.0930 - accuracy: 0.9936 - val_loss: 0.8099 - val_accuracy: 0.8364\n",
      "Epoch 63/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.1098 - accuracy: 0.9876 - val_loss: 0.8289 - val_accuracy: 0.8502\n",
      "Epoch 64/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.0934 - accuracy: 0.9934 - val_loss: 0.9265 - val_accuracy: 0.7908\n",
      "Epoch 65/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.0903 - accuracy: 0.9935 - val_loss: 0.8840 - val_accuracy: 0.8325\n",
      "Epoch 66/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.0956 - accuracy: 0.9917 - val_loss: 0.8385 - val_accuracy: 0.8449\n",
      "Epoch 67/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.0955 - accuracy: 0.9926 - val_loss: 0.9304 - val_accuracy: 0.8004\n",
      "Epoch 68/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.0886 - accuracy: 0.9937 - val_loss: 0.9751 - val_accuracy: 0.8046\n",
      "Epoch 69/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.0907 - accuracy: 0.9931 - val_loss: 0.8468 - val_accuracy: 0.8177\n",
      "Epoch 70/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.0927 - accuracy: 0.9925 - val_loss: 0.8311 - val_accuracy: 0.8367\n",
      "Epoch 71/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.0947 - accuracy: 0.9917 - val_loss: 0.8589 - val_accuracy: 0.8431\n",
      "Epoch 72/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.0924 - accuracy: 0.9932 - val_loss: 0.8720 - val_accuracy: 0.8435\n",
      "Epoch 73/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.0930 - accuracy: 0.9933 - val_loss: 0.8380 - val_accuracy: 0.8336\n",
      "Epoch 74/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.0904 - accuracy: 0.9928 - val_loss: 1.2357 - val_accuracy: 0.7329\n",
      "Epoch 75/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.0920 - accuracy: 0.9923 - val_loss: 0.8792 - val_accuracy: 0.8311\n",
      "Epoch 76/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.1014 - accuracy: 0.9900 - val_loss: 0.7795 - val_accuracy: 0.8258\n",
      "Epoch 77/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.0890 - accuracy: 0.9942 - val_loss: 0.8498 - val_accuracy: 0.8318\n",
      "Epoch 78/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.0948 - accuracy: 0.9924 - val_loss: 1.0895 - val_accuracy: 0.8371\n",
      "Epoch 79/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.0910 - accuracy: 0.9930 - val_loss: 0.9785 - val_accuracy: 0.7965\n",
      "Epoch 80/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.0896 - accuracy: 0.9927 - val_loss: 0.8295 - val_accuracy: 0.8382\n",
      "Epoch 81/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.0943 - accuracy: 0.9913 - val_loss: 0.8157 - val_accuracy: 0.8329\n",
      "Epoch 82/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.0989 - accuracy: 0.9904 - val_loss: 0.8344 - val_accuracy: 0.8095\n",
      "Epoch 83/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.0865 - accuracy: 0.9948 - val_loss: 0.9038 - val_accuracy: 0.8466\n",
      "Epoch 84/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.0942 - accuracy: 0.9918 - val_loss: 0.8348 - val_accuracy: 0.8396\n",
      "Epoch 85/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.0901 - accuracy: 0.9934 - val_loss: 0.8590 - val_accuracy: 0.8473\n",
      "Epoch 86/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.0901 - accuracy: 0.9921 - val_loss: 0.8445 - val_accuracy: 0.8201\n",
      "Epoch 87/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.0940 - accuracy: 0.9919 - val_loss: 0.8745 - val_accuracy: 0.8396\n",
      "Epoch 88/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.0843 - accuracy: 0.9947 - val_loss: 0.8800 - val_accuracy: 0.8382\n",
      "Epoch 89/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.0846 - accuracy: 0.9934 - val_loss: 0.8854 - val_accuracy: 0.8205\n",
      "Epoch 90/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.0902 - accuracy: 0.9917 - val_loss: 0.8923 - val_accuracy: 0.8230\n",
      "Epoch 91/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.0890 - accuracy: 0.9933 - val_loss: 0.8656 - val_accuracy: 0.8470\n",
      "Epoch 92/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.0906 - accuracy: 0.9930 - val_loss: 0.8832 - val_accuracy: 0.8198\n",
      "Epoch 93/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.0803 - accuracy: 0.9959 - val_loss: 0.9206 - val_accuracy: 0.8488\n",
      "Epoch 94/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.0919 - accuracy: 0.9920 - val_loss: 0.9943 - val_accuracy: 0.8032\n",
      "Epoch 95/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.0875 - accuracy: 0.9938 - val_loss: 0.8665 - val_accuracy: 0.8279\n",
      "Epoch 96/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.0849 - accuracy: 0.9944 - val_loss: 0.8590 - val_accuracy: 0.8470\n",
      "Epoch 97/100\n",
      "10010/10010 [==============================] - 12s 1ms/step - loss: 0.0940 - accuracy: 0.9900 - val_loss: 0.9491 - val_accuracy: 0.8042\n",
      "Epoch 98/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.0869 - accuracy: 0.9929 - val_loss: 0.9397 - val_accuracy: 0.8148\n",
      "Epoch 99/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.0855 - accuracy: 0.9946 - val_loss: 0.8610 - val_accuracy: 0.8449\n",
      "Epoch 100/100\n",
      "10010/10010 [==============================] - 13s 1ms/step - loss: 0.0934 - accuracy: 0.9898 - val_loss: 0.8570 - val_accuracy: 0.8346\n"
     ]
    }
   ],
   "source": [
    "history = model.fit([train_text_matrix, train_image_matrix],train_label,validation_data=([test_text_matrix,test_image_matrix],test_label),batch_size =32,epochs=100,callbacks=callbacks_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('XL_gossip_history.json', 'w') as f:\n",
    "    json.dump(str(history.history), f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "'acc'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-34-534a50a6ca6c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mmatplotlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpyplot\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'acc'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'val_acc'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtitle\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Model accuracy'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mylabel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Accuracy'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 'acc'"
     ]
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.plot(history.history['acc'])\n",
    "plt.plot(history.history['val_acc'])\n",
    "plt.title('Model accuracy')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.xlabel('Epoch')\n",
    "plt.legend(['Train', 'Test'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.plot(history.history['loss'])\n",
    "plt.plot(history.history['val_loss'])\n",
    "plt.title('Model Loss')\n",
    "plt.ylabel('Loss')\n",
    "plt.xlabel('Epoch')\n",
    "plt.legend(['Train', 'Test'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### XLNET + dense"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def baseline_model():\n",
    "    model = Sequential()\n",
    "    model.add(Flatten(input_shape=(50,768)))\n",
    "    model.add(Dense(1000, activation='relu', kernel_regularizer=regularizers.l2(0.01), kernel_initializer=initializers.he_normal(seed=0)))\n",
    "    #model.add(Dropout(0.4))\n",
    "    model.add(Dense(500, activation='relu', kernel_regularizer=regularizers.l2(0.01), kernel_initializer=initializers.he_normal(seed=0)))\n",
    "    #model.add(Dropout(0.4))\n",
    "    model.add(Dense(100, activation='relu', kernel_regularizer=regularizers.l2(0.01), kernel_initializer=initializers.he_normal(seed=0)))\n",
    "    #model.add(BatchNormalization())\n",
    "    model.add(Dropout(0.4))\n",
    "    model.add(Dense(2, activation='softmax'))\n",
    "    #adam = optimizers.Adam(lr=1e-4)\n",
    "    #adamax = optimizers.Adamax(learning_rate=0.002, beta_1=0.9, beta_2=0.999)\n",
    "    sgd = optimizers.SGD(lr=0.01, clipnorm=1.)\n",
    "    #adadelta = optimizers.Adadelta(lr=1.0, rho=0.95)\n",
    "    model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoint = ModelCheckpoint(filepath='/home/karmanya/finalExperiment/checkpoints_gossip/dense_Text_model.hdf5', monitor='val_acc', verbose=1, save_best_only=True, mode='max')\n",
    "callbacks_list = [checkpoint]\n",
    "model = baseline_model()\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "textHistory = model.fit(train_text_matrix,train_label,validation_data=(test_text_matrix,test_label),batch_size =32,epochs =100,callbacks=callbacks_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"import matplotlib.pyplot as plt\n",
    "plt.plot(textHistory.history['acc'])\n",
    "plt.plot(textHistory.history['val_acc'])\n",
    "plt.title('Model accuracy')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.xlabel('Epoch')\n",
    "plt.legend(['Train', 'Test'], loc='upper left')\n",
    "plt.show()\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Conv1D,MaxPooling1D,Flatten\n",
    "from keras.layers import LSTM, Bidirectional\n",
    "from keras.optimizers import RMSprop"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### XLNET+ LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lstm():\n",
    "    model = Sequential()\n",
    "    model.add(LSTM(128,input_shape=(50,768)))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(595, activation='relu'))\n",
    "    model.add(Dense(2, activation='softmax'))\n",
    "    model.compile(loss='categorical_crossentropy',optimizer='sgd',metrics=['accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoint = ModelCheckpoint(filepath='/home/karmanya/finalExperiment/checkpoints_gossip/lstm_text_model.hdf5', monitor='val_acc', verbose=1, save_best_only=True, mode='max')\n",
    "callbacks_list = [checkpoint]\n",
    "model = lstm()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.fit(train_text_matrix,train_label,validation_data=(test_text_matrix,test_label),batch_size =32,epochs =100,callbacks=callbacks_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### XLNET + CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cnn_model():\n",
    "    model = Sequential()\n",
    "    model.add(Conv1D(filters=3, kernel_size=5, activation='relu',data_format='channels_first' , input_shape=(50,768)))\n",
    "    model.add(MaxPooling1D(pool_size=2))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Conv1D(filters=3, kernel_size=5, activation='relu',data_format='channels_first' ))\n",
    "    model.add(MaxPooling1D(pool_size=2))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(500, activation='relu'))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(100, activation='relu'))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(2, activation='softmax'))\n",
    "    model.compile(loss='categorical_crossentropy', optimizer='sgd', metrics=['accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model=cnn_model()\n",
    "checkpoint = ModelCheckpoint(filepath='/home/karmanya/finalExperiment/checkpoints_gossip/cnn_text_model.hdf5', monitor='val_acc', verbose=1, save_best_only=True, mode='max')\n",
    "callbacks_list = [checkpoint]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.fit(train_text_matrix,train_label,validation_data=(test_text_matrix,test_label),batch_size =32,epochs =100, callbacks=callbacks_list)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (BERT)",
   "language": "python",
   "name": "bert"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}