EmbeddingCat.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import os\n",
    "import math\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn import preprocessing\n",
    "from scipy.spatial import distance\n",
    "import matplotlib.pyplot as plt\n",
    "np.random.seed(42)\n",
    "import keras\n",
    "from keras.models import Model, Sequential\n",
    "from keras.layers import Embedding, Dense, Dropout, Input, Concatenate, concatenate, Flatten, Reshape, Lambda\n",
    "from keras.callbacks import EarlyStopping\n",
    "from keras.utils import plot_model\n",
    "from sklearn.model_selection import train_test_split as splt\n",
    "from sklearn.cluster import KMeans\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"2\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def step1(age, sex, chd=True):\n",
    "    # CHD\n",
    "    if chd:  # constants for chd\n",
    "        if sex:  # 0 for men and 1 for women\n",
    "            a = -29.8\n",
    "            p = 6.36\n",
    "        else:\n",
    "            a = -22.1\n",
    "            p = 4.71\n",
    "\n",
    "    else:  # constants for non chd\n",
    "        if sex:\n",
    "            a = -31.0\n",
    "            p = 6.62\n",
    "        else:\n",
    "\n",
    "            a = -26.7\n",
    "            p = 5.64\n",
    "\n",
    "    # print(\"a =\", a, \"; p =\",p)\n",
    "\n",
    "    s = math.exp(-(math.exp(a)) * (age - 20) ** p)\n",
    "    return s\n",
    "\n",
    "\n",
    "def step2(chol, SBP, smoker, chd=True):\n",
    "    if chd:\n",
    "        c_smoker = 0.71\n",
    "        c_chol = 0.24\n",
    "        c_SBP = 0.018\n",
    "    else:\n",
    "        c_smoker = 0.63\n",
    "        c_chol = 0.02\n",
    "        c_SBP = 0.022\n",
    "\n",
    "    w = (c_chol * (chol - 6)) + (c_SBP * (SBP - 120)) + (c_smoker * smoker)\n",
    "    return w\n",
    "\n",
    "\n",
    "def score_algorithm(age, chol, SBP, sex, smoker):\n",
    "    # CHD\n",
    "    smoker = 1 if smoker >= 1 else 0\n",
    "\n",
    "    \n",
    "    s = step1(age, sex)\n",
    "    s10 = step1(age + 10, sex)\n",
    "\n",
    "    w = step2(chol, SBP, smoker)\n",
    "    \n",
    "\n",
    "    s = s ** (math.exp(w))\n",
    "    s10 = s10 ** (math.exp(w))\n",
    "    try:\n",
    "        stot = s10 / s\n",
    "    except:\n",
    "        stot = 1\n",
    "    riskc = 1 - stot\n",
    "\n",
    "    # NON CHD\n",
    "    s = step1(age, sex, chd=False)\n",
    "    s10 = step1(age + 10, sex, chd=False)\n",
    "\n",
    "    w = step2(chol, SBP, smoker, chd=False)\n",
    "\n",
    "    s = s ** (math.exp(w))\n",
    "    s10 = s10 ** (math.exp(w))\n",
    "    try:\n",
    "        stot = s10 / s\n",
    "    except:\n",
    "        stot = 1\n",
    "    risknon = 1 - stot\n",
    "\n",
    "    # print (\"risk CHD: \", riskc *100)\n",
    "    # print (\"risk nonCHD: \" ,risknon * 100)\n",
    "    risktot = 1 - (1 - riskc) * (1 - risknon)\n",
    "\n",
    "    # print('total RISK:',risktot)\n",
    "    return risktot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_history(history):\n",
    "    loss_list = [s for s in history.history.keys() if 'loss' in s and 'val' not in s]\n",
    "    val_loss_list = [s for s in history.history.keys() if 'loss' in s and 'val' in s]\n",
    "    acc_list = [s for s in history.history.keys() if 'acc' in s and 'val' not in s]\n",
    "    val_acc_list = [s for s in history.history.keys() if 'acc' in s and 'val' in s]\n",
    "\n",
    "    if len(loss_list) == 0:\n",
    "        print('Loss is missing in history')\n",
    "        return \n",
    "    \n",
    "    ## As loss always exists\n",
    "    epochs = range(1,len(history.history[loss_list[0]]) + 1)\n",
    "    \n",
    "    ## Loss\n",
    "    plt.figure(1)\n",
    "    for l in loss_list:\n",
    "        plt.plot(epochs, history.history[l], 'b', label='Training loss (' + str(str(format(history.history[l][-1],'.5f'))+')'))\n",
    "    for l in val_loss_list:\n",
    "        plt.plot(epochs, history.history[l], 'g', label='Validation loss (' + str(str(format(history.history[l][-1],'.5f'))+')'))\n",
    "    \n",
    "    plt.title('Loss')\n",
    "    plt.xlabel('Epochs')\n",
    "    plt.ylabel('Loss')\n",
    "    plt.legend()\n",
    "    \n",
    "    ## Accuracy\n",
    "    plt.figure(2)\n",
    "    for l in acc_list:\n",
    "        plt.plot(epochs, history.history[l], 'b', label='Training accuracy (' + str(format(history.history[l][-1],'.5f'))+')')\n",
    "    for l in val_acc_list:    \n",
    "        plt.plot(epochs, history.history[l], 'g', label='Validation accuracy (' + str(format(history.history[l][-1],'.5f'))+')')\n",
    "    \n",
    "    plt.title('Accuracy')\n",
    "    plt.xlabel('Epochs')\n",
    "    plt.ylabel('Accuracy')\n",
    "    plt.legend()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfx = pd.read_csv(\"Data/plic_milano_clean.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "i = 600\n",
    "age = dfx['ana:age'][i]\n",
    "chol = dfx['lab:total_cholesterol'][i] * 0.02586\n",
    "pas = min(float(dfx['esa_obi:sbp'][i]), 1000)\n",
    "sex = dfx['ana:gender'][i]\n",
    "smoker = dfx['ana_fis:smoking_recod'][i]\n",
    "\n",
    "print((dfx['lab:hdl'] + dfx['lab:calculated_ldl']).mean())\n",
    "print(dfx['lab:total_cholesterol'].mean())\n",
    "print()\n",
    "\n",
    "print(age)\n",
    "print(chol)\n",
    "print(pas)\n",
    "print(sex)\n",
    "print(smoker)\n",
    "print(score_algorithm(age, chol, pas, sex, smoker))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = np.zeros(dfx.shape[0])\n",
    "for i, visit in dfx.iterrows():\n",
    "    age = dfx['ana:age'][i]\n",
    "    chol = dfx['lab:total_cholesterol'][i] * 0.02586\n",
    "    pas = float(dfx['esa_obi:sbp'][i])\n",
    "    sex = dfx['ana:gender'][i]\n",
    "    smoker = dfx['ana_fis:smoking_recod'][i]\n",
    "    \n",
    "    y[i] = score_algorithm(age, chol, pas, sex, smoker)\n",
    "\n",
    "# Thresholds are 0 - 0.01, 0.01 - 0.02, 0.02 - 0.05, 0.05 - 1    \n",
    "\n",
    "y_cat = np.zeros((y.shape[0], 4))\n",
    "stats = np.zeros(y.shape[0])\n",
    "for i, val in enumerate(y):\n",
    "    if(val < 0.01): # No\n",
    "        y_cat[i][0] = 1\n",
    "    elif(val < 0.02): # Low\n",
    "        y_cat[i][1] = 1\n",
    "        stats[i] = 1\n",
    "    elif(val < 0.05): # Medium\n",
    "        y_cat[i][2] = 1\n",
    "        stats[i] = 2\n",
    "    else:             # High \n",
    "        y_cat[i][3] = 1\n",
    "        stats[i] = 3\n",
    "        \n",
    "y_cat.shape\n",
    "\n",
    "plt.hist(stats)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfx = dfx.drop(labels = ['lab:total_cholesterol', \"esa_obi:sbp\", \"ana_fis:smoking_recod\"], axis=1)\n",
    "dfx = dfx.dropna(how='any', axis=1)\n",
    "dfx = dfx.select_dtypes(include=['int', 'float'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "threshold = 10\n",
    "dfxInt = dfx.select_dtypes(include=['int'])\n",
    "dfxCat = dfxInt[dfxInt.columns[dfxInt.max()<=threshold]]\n",
    "dfxCont = dfx[dfx.columns[dfx.max()>threshold]]\n",
    "#dfxCont = dfxInt[dfxInt.columns[dfxInt.max()>threshold]] + dfx.select_dtypes(include=['float'])\n",
    "\n",
    "# dfxCat.describe()\n",
    "# dfxCont.describe()\n",
    "print(dfxCat.shape)\n",
    "print(dfxCont.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xCat = dfx.apply(lambda r: str(r.values)[1:-1].replace(\"\\n\", \"\").replace(\" \", \"\"), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xCat = dfxCat.values\n",
    "xCont = dfxCont.values\n",
    "xCont = xCont / xCont.max(axis=0)\n",
    "\n",
    "print(\"cat:\", xCat.shape)\n",
    "print(\"cont:\", xCont.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "H_cat = distance.pdist(xCat, 'hamming')\n",
    "H_cat_dense = distance.squareform(H_cat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "kmeans = KMeans(n_clusters=5, precompute_distances=False, algorithm='elkan', random_state=42)\n",
    "kmeans.fit(H_cat_dense)\n",
    "\n",
    "for i in range(5):\n",
    "    print(\"{}: {}\".format(i, np.where(kmeans.labels_ == i)[0].shape))\n",
    "\n",
    "kmeans.labels_[kmeans.labels_ == 3] = 2\n",
    "kmeans.labels_[kmeans.labels_ == 4] = 3\n",
    "print()\n",
    "\n",
    "for i in range(4):\n",
    "    print(\"{}: {}\".format(i, np.where(kmeans.labels_ == i)[0].shape))\n",
    "\n",
    "xCat = kmeans.labels_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xCat = (xCat)[..., np.newaxis]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers.core import Dropout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "catIn = Input(shape=(xCat.shape[1],))\n",
    "\n",
    "## Branch for Continuos Values \n",
    "# contBranch = Dense(16, activation='sigmoid')(contIn)\n",
    "# contBranch = Dropout(0.1)(contBranch)\n",
    "# contBranch = Dense(16, activation='sigmoid')(contBranch)\n",
    "\n",
    "catBranch = Embedding(5, 100, input_length=1)(catIn)\n",
    "catBranch = Flatten()(catBranch)\n",
    "catBranch = Dense(16, activation='sigmoid')(catBranch)\n",
    "# catBranch = Dropout(0.1)(catBranch)\n",
    "# catBranch = Dense(16, activation='sigmoid')(catBranch)\n",
    "\n",
    "# concat = Concatenate()([contBranch, catBranch])\n",
    "\n",
    "fcnn = Dense(32, activation='sigmoid')(catBranch)\n",
    "fcnn = Dense(16, activation='sigmoid')(fcnn)\n",
    "fcnn = Dense(8, activation='sigmoid')(fcnn)\n",
    "\n",
    "output = Dense(4, activation='softmax')(fcnn)\n",
    "\n",
    "model = Model(inputs = [catIn], outputs = [output])\n",
    "\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "\n",
    "# model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_Cat_tr, x_Cat_ts, x_Cont_tr, x_Cont_ts, y_tr, y_ts, stat_tr, stat_ts = splt(xCat, xCont, y_cat, stats, test_size=0.25, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# es = EarlyStopping(patience = 100, restore_best_weights = True)\n",
    "history = model.fit(x=[x_Cat_tr], y=y_tr, \n",
    "                    validation_data=([x_Cat_ts], y_ts), \n",
    "                    epochs=500, verbose=1, \n",
    "#                     callbacks = [es]\n",
    "                   )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_history(history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bokeh.plotting import figure, output_notebook, show\n",
    "output_notebook()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras import backend as K\n",
    "\n",
    "def get_activations(model, layer, X_batch):\n",
    "    activations_f = K.function([model.layers[0].input, K.learning_phase()], [layer.output,])\n",
    "    activations = activations_f((X_batch, False))\n",
    "    return activations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings_tr = get_activations(model, model.layers[1], X_batch=x_Cat_tr)[0]\n",
    "embeddings_ts = get_activations(model, model.layers[1], X_batch=x_Cat_ts)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings_tr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scale = 300"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings_tr.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings_tr = embeddings_tr.reshape((4335, 100, 1))\n",
    "embeddings_tr.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.manifold import MDS, TSNE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sklearn\n",
    "sklearn.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.spatial.distance import squareform, pdist    \n",
    "\n",
    "embeddings_tr_sim = squareform(pdist(embeddings_tr[..., 0],'euclidean'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tsne = TSNE(n_components=2)\n",
    "embeddings_tr_tsne = tsne.fit_transform(embeddings_tr[:, :, 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "colors_map = np.argmax(y_tr, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "colors_map[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "colors = [x for x in 'yellow-orange-red-black-magenta-black-purple-coral-lime'.split('-')]\n",
    "\n",
    "p = figure(plot_width=600, plot_height=600)\n",
    "# p.circle(embeddings_tr_tsne[:, 0]*scale, embeddings_tr_tsne[:, 1]*scale, size=4, alpha=0.4)\n",
    "for cl in range(4):\n",
    "    indices = np.where(colors_map==cl)[0]\n",
    "    p.circle(embeddings_tr_tsne[indices, 0]*scale*100, embeddings_tr_tsne[indices, 1]*scale*100, \n",
    "             color=colors[cl], size=20, alpha=0.3)\n",
    "\n",
    "show(p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fn = \"_embedded_cat_model_improved_\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(\"Models/model\" + fn + \".h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(history, open(\"History/history\" + fn + \".pkl\", \"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import keras\n",
    "import pydot as pyd\n",
    "from IPython.display import Image\n",
    "from keras.utils.vis_utils import model_to_dot\n",
    "\n",
    "keras.utils.vis_utils.pydot = pyd\n",
    "\n",
    "#Visualize Model\n",
    "\n",
    "def visualize_model(model):\n",
    "    return Image(model_to_dot(model).create(prog='dot', format='png'))\n",
    "#create your model\n",
    "#then call the function on your model\n",
    "visualize_model(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:score]",
   "language": "python",
   "name": "conda-env-score-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}