10HM_Robustness_tests.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "6809fc9b-2439-41bd-bb2d-be3ddbd80f0c",
   "metadata": {},
   "source": [
    "# Robustness tests\n",
    "\n",
    "For this notebook **you need to run the 4M and 4H notbeooks previously!!**.\n",
    "\n",
    "In this notebook we are going to analyze with different tests the robustness and replicability of the results obtained by the algorithm of cluster annotation."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c141bbcb-caa9-4251-bbe9-7dcad3adcd80",
   "metadata": {},
   "source": [
    "## imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d89d457d-3389-4ba4-9621-5ffa9d078a63",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f0805230-74fb-499b-b94e-b068988209dc",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import scanpy as sc\n",
    "import scanpy.external as sce\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import triku as tk\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib as mpl\n",
    "from tqdm.notebook import tqdm\n",
    "import scipy.sparse as spr\n",
    "import matplotlib.cm as cm\n",
    "import matplotlib.pylab as pylab\n",
    "import networkx as nx\n",
    "import functools\n",
    "import math"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01b4c3bf-fd37-4f0a-989c-509c08f16c0b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!pip install munkres\n",
    "from munkres import Munkres"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a7f3f9f8-3379-4834-a032-fd713ea658be",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# local imports and imports from other notebooks\n",
    "from cellassign import assign_cats\n",
    "from fb_functions import make_gene_scoring_with_expr, plot_score_graph, plot_UMAPS_gene, plot_adata_cluster_properties\n",
    "%store -r dict_colors_human\n",
    "%store -r dict_colors_mouse\n",
    "\n",
    "dict_colors_human_mouse = {**dict_colors_human , **dict_colors_mouse}\n",
    "\n",
    "%store -r seed\n",
    "%store -r magma\n",
    "%store -r data_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c6d265a9-6866-4952-b53a-0c3fc9f2aeb3",
   "metadata": {},
   "outputs": [],
   "source": [
    "magma2 = mpl.colors.LinearSegmentedColormap.from_list(\"\", [plt.get_cmap('magma')(i) for i in np.linspace(0,1, 80)][:70])\n",
    "\n",
    "%store magma2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "36fd2f24-365a-46f3-a5a6-9413cec73218",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%store -r plot_params\n",
    "\n",
    "pylab.rcParams.update(plot_params)\n",
    "pd.set_option('display.max_columns', None)\n",
    "pd.options.display.float_format = \"{:,.2f}\".format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fffd51c8-1f48-4dc1-8c2b-c49e924f82e9",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# we are going to use dicts from 3H/3M and not 4H/4M because we want to replicate the results \n",
    "# from that notebook (in 4H/4M the clusters are not assigned again, only the genes are shown)\n",
    "\n",
    "%store -r dict_cats_clusters_robust_3H \n",
    "%store -r dict_cats_clusters_robust_3M"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8a868ed2-4d74-415a-a4d7-a379cf6ea675",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%store -r list_all_datasets_human\n",
    "%store -r list_all_datasets_mouse\n",
    "\n",
    "%store -r list_names_human\n",
    "%store -r list_names_mouse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0c1f4bce-695f-49a8-b68d-a6616415d4ea",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def change_texts_format(ax):\n",
    "    for text in ax.texts:\n",
    "            if text.get_text() == '1.00':\n",
    "                text.set_text('1')\n",
    "                \n",
    "            elif '0.' in text.get_text():\n",
    "                text.set_text(text.get_text().replace('0.', '.'))\n",
    "                \n",
    "def round_df(df, N=2):\n",
    "    return df.round(N).div(df.round(N).sum(axis=1), axis=0).round(N)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9e2f156c-472b-4171-9d4b-3178af79ab0b",
   "metadata": {},
   "source": [
    "# Stratified bootstrap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "73cd91f1-c3bb-44d5-96e7-e2f8a261ffff",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dir_save_strat = 'results/bootstrap_cell_types'\n",
    "os.makedirs(dir_save_strat, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d9d0da23-9939-4df1-a100-641479d6d22a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def stratified_subsampling(adata, obs_clusters, frac, seed):\n",
    "    idx_sub = adata.obs.groupby(obs_clusters, group_keys=False).apply(lambda x: x.sample(frac=frac, random_state=seed)).index.values\n",
    "    adata_sub = adata[idx_sub].copy()\n",
    "    return adata_sub\n",
    "\n",
    "def bootstrap_cluster_assign(adata, obs_clusters, frac, N_iters, dict_cats_cluster):\n",
    "    df_bootstrap = pd.DataFrame(columns = [-1] + list(range(N_iters)), index=adata.obs_names)\n",
    "    df_bootstrap.loc[adata.obs_names, -1] = adata.obs['cluster_robust']\n",
    "\n",
    "    for it in range(N_iters):\n",
    "        try:\n",
    "            print(adata.obs['Author'].values[0], int(adata.obs['Year'].values[0]), it)\n",
    "            adata_sub = stratified_subsampling(adata=adata.copy(), obs_clusters=obs_clusters, frac=frac, seed=it)\n",
    "\n",
    "    #         We preprocess the adata again\n",
    "            sc.pp.filter_genes(adata_sub, min_counts=1)\n",
    "            sc.pp.pca(adata_sub, random_state=seed, n_comps=50)\n",
    "            if 'X_pca_harmony' in adata_sub.obsm:\n",
    "                use_rep = 'X_pca_harmony_sub'\n",
    "                sce.pp.harmony_integrate(adata_sub, key='Internal sample identifier', \n",
    "                                         max_iter_harmony=50, adjusted_basis=use_rep)\n",
    "\n",
    "            else:\n",
    "                use_rep = 'X_pca'\n",
    "\n",
    "            n_neighbors = int(adata_sub.uns['neighbors']['params']['n_neighbors'] * frac)\n",
    "\n",
    "            sc.pp.neighbors(adata_sub, use_rep=use_rep,  n_neighbors=n_neighbors, metric='cosine')\n",
    "            tk.tl.triku(adata_sub, use_raw=False)\n",
    "\n",
    "            sc.pp.pca(adata_sub, random_state=seed, n_comps=50)\n",
    "            if 'X_pca_harmony' in adata_sub.obsm:\n",
    "                sce.pp.harmony_integrate(adata_sub, key='Internal sample identifier', \n",
    "                                         max_iter_harmony=50, adjusted_basis=use_rep)\n",
    "\n",
    "            sc.pp.neighbors(adata_sub, use_rep=use_rep,  n_neighbors=n_neighbors, metric='cosine')\n",
    "\n",
    "            sc.tl.leiden(adata_sub, resolution=frac * adata.uns['leiden']['params']['resolution'], \n",
    "                        key_added='leiden_sub')  # I know that this is not linear, but it is an approach\n",
    "\n",
    "\n",
    "            min_score = adata_sub.uns['cell_assign'][obs_clusters]['min_score']\n",
    "            quantile_gene_sel = adata_sub.uns['cell_assign'][obs_clusters]['quantile_gene_sel']\n",
    "            assign_cats(adata_sub, column_groupby='leiden_sub', dict_cats=dict_cats_cluster, min_score=min_score, \n",
    "                        quantile_gene_sel=quantile_gene_sel, \n",
    "                        key_added=obs_clusters + '_sub', others_name='U', verbose=False)\n",
    "            df_bootstrap.loc[adata_sub.obs_names, it] = adata_sub.obs[obs_clusters + '_sub']\n",
    "        except:\n",
    "            continue\n",
    "    return df_bootstrap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fc7beaed-af84-4d11-9ea0-c4551a932f43",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "all_frac_vals = [0.5, 0.7, 0.8, 0.95, 0.99]\n",
    "frac = 0.8\n",
    "N_iters = 30\n",
    "\n",
    "for frac in all_frac_vals:\n",
    "    for adata in list_all_datasets_human:\n",
    "        org = 'human'\n",
    "        name = f\"{adata.obs['Author'].values[0]}_{int(adata.obs['Year'].values[0])}_{org}_frac-{frac}_Niters-{N_iters}.csv\"\n",
    "        if name not in os.listdir(dir_save_strat):\n",
    "            df_bootstrap = bootstrap_cluster_assign(adata, obs_clusters='cluster_robust', \n",
    "                                                        frac=frac, N_iters=N_iters, \n",
    "                                                        dict_cats_cluster=dict_cats_clusters_robust_3H)\n",
    "\n",
    "            df_bootstrap = df_bootstrap.sort_values(by=-1)\n",
    "            df_bootstrap.to_csv(f\"{dir_save_strat}/{name}\", sep='\\t')\n",
    "\n",
    "\n",
    "\n",
    "    for adata in list_all_datasets_mouse:\n",
    "        org = 'mouse'\n",
    "        name = f\"{adata.obs['Author'].values[0]}_{int(adata.obs['Year'].values[0])}_{org}_frac-{frac}_Niters-{N_iters}.csv\"\n",
    "        if name not in os.listdir(dir_save_strat):\n",
    "            df_bootstrap = bootstrap_cluster_assign(adata, obs_clusters='cluster_robust', \n",
    "                                                        frac=frac, N_iters=N_iters, \n",
    "                                                        dict_cats_cluster=dict_cats_clusters_robust_3M)\n",
    "\n",
    "            df_bootstrap = df_bootstrap.sort_values(by=-1)\n",
    "            df_bootstrap.to_csv(f\"{dir_save_strat}/{name}\", sep='\\t')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d2fc8aba-0134-40ea-b5fa-9c2d38c8c8bc",
   "metadata": {},
   "source": [
    "## Load dict_bootstraps (with all options)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7a2f7637-84b1-45b6-9074-009114541fc1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def read_dict_bootstrap(dir_save='results/bootstrap_cell_types', not_include=[]):\n",
    "    list_files = sorted([i for i in os.listdir(dir_save) if i[-3:] == 'csv'])\n",
    "    \n",
    "    dict_bootstraps = {}\n",
    "\n",
    "    for file in list_files:\n",
    "        include = True\n",
    "        for not_in in not_include:\n",
    "            if not_in in file:\n",
    "                include = False\n",
    "                \n",
    "        if include:        \n",
    "            df = pd.read_csv(f\"{dir_save}/{file}\", sep='\\t', index_col=0)\n",
    "            repls = ('.csv', ''), ('frac-', ''), ('Niters-', '')\n",
    "            file_red = functools.reduce(lambda a, kv: a.replace(*kv), repls, file)\n",
    "            author, year, org, frac, N_iters = file_red.split('_')\n",
    "\n",
    "            dict_bootstraps[f\"{author}_{year}_{org}_{frac}_{N_iters}\"] = df\n",
    "    \n",
    "    return dict_bootstraps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "74a9d892-54f9-4474-82c3-9b723c807a78",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dict_bootstraps = read_dict_bootstrap()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7c29ad45-1b91-4089-9474-fe7965acc475",
   "metadata": {},
   "source": [
    "## Calculating and plotting the scores for a cell being assigned the same cluster "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8892ee55-7e0b-4963-9a9d-f43fb46cc242",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def assign_cluster_score(df_bootstrap, N_iters):\n",
    "    \"\"\"\n",
    "    Creates a column with the proportion of iterations that have the same label assigned\n",
    "    as the original labelling. Since there are NaN values, the denominator is the\n",
    "    number of iterations that have non NaN values.\n",
    "    \"\"\"\n",
    "\n",
    "    cols = [str(i) for i in range(N_iters)]\n",
    "    \n",
    "    \n",
    "    is_equal_to_val = np.zeros((len(df_bootstrap), N_iters), np.bool)\n",
    "    is_na = np.zeros((len(df_bootstrap), N_iters), np.bool)\n",
    "    \n",
    "    for it in range(N_iters):\n",
    "        is_equal_to_val[:, it] = df_bootstrap.loc[:, str(it)].values == df_bootstrap.loc[:, '-1'].values\n",
    "        is_na[:, it] = pd.isna(df_bootstrap.loc[:, str(it)].values)\n",
    "        \n",
    "    df_bootstrap[f'same_cluster_prop'] = is_equal_to_val.sum(1)/(N_iters - is_na.sum(1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "149de4e9-24bf-4f54-bfeb-53b59bd308e4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "frac_vals = ['0.5', '0.7', '0.8', '0.95', '0.99']\n",
    "N_iters = 30\n",
    "\n",
    "for df_bootstrap in dict_bootstraps.values():\n",
    "    assign_cluster_score(df_bootstrap, N_iters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6604f7d9-217a-49fc-a9e3-b3d8a6b5754f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Create a dataframe, per dataset, that contains all the different fraction scores.\n",
    "# This will allow us to compare values and to see tendencies\n",
    "\n",
    "dict_df_score_fracs = {}\n",
    "\n",
    "for idx, key in enumerate(list_names_human + list_names_mouse):\n",
    "    org = 'human' if idx < len(list_names_human) else 'mouse'\n",
    "    \n",
    "    \n",
    "    name, year = ' '.join(key.split(' ')[:-1]), key.split(' ')[-1]\n",
    "    df_scores = pd.DataFrame(index=(list_all_datasets_human + list_all_datasets_mouse)[idx].obs_names, \n",
    "                             columns=['cluster'] + frac_vals)\n",
    "    \n",
    "    for frac in frac_vals:\n",
    "        df_vals = dict_bootstraps[f\"{name}_{year}_{org}_{frac}_{N_iters}\"]\n",
    "        \n",
    "        \n",
    "        if frac == frac_vals[0]:\n",
    "            df_scores['cluster'] = df_vals.loc[df_scores.index, \"-1\"].values\n",
    "        \n",
    "        df_scores[frac] = df_vals.loc[df_scores.index, \"same_cluster_prop\"].values\n",
    "    \n",
    "    dict_df_score_fracs[f\"{name}_{year}_{org}_{N_iters}\"] = df_scores      \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6c909727-d04d-4519-852f-02eb1a4d47ba",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for adata, df_score in zip(list_all_datasets_human+list_all_datasets_mouse, dict_df_score_fracs.values()):\n",
    "    for frac in frac_vals:\n",
    "        adata.obs[f\"bootstrap_{frac}\"] = df_score[frac]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3eedbadb-9da1-458e-8d95-ff4f3532dc1f",
   "metadata": {},
   "source": [
    "### Plotting the score (UMAP, humans)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "57dd79a5-9482-4846-b4dc-0106627f85be",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for frac in ['cluster_robust'] + ['bootstrap_' + i for i in frac_vals]:\n",
    "    print('FRAC', frac)\n",
    "    plot_UMAPS_gene(frac, list_datasets=list_all_datasets_human, list_names=list_names_human, n_cols=5, cmap=magma2)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4446c8bc-1573-4cbc-abca-d53fa8562819",
   "metadata": {},
   "source": [
    "### Plotting the score (UMAP, mouse)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2bcc7101-9bd9-4cee-8f8d-cf9998513675",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for frac in ['cluster_robust'] + ['bootstrap_' + i for i in frac_vals]:\n",
    "    print('FRAC', frac)\n",
    "    plot_UMAPS_gene(frac, list_datasets=list_all_datasets_mouse, list_names=list_names_mouse, n_cols=3, cmap=magma2)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0c51f3f8-e7e4-4a12-9ec5-802d1a230d0c",
   "metadata": {},
   "source": [
    "## Distribution plots of score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b32d57c7-08a3-43c8-8391-e200253d22ab",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dict_df_score_fracs_human = {key: val for (key, val) in dict_df_score_fracs.items() if 'human' in key}\n",
    "dict_df_score_fracs_mouse = {key: val for (key, val) in dict_df_score_fracs.items() if 'mouse' in key}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bdf006d0-f347-4804-86df-f70737a0a52e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def plot_distributions_bootstrap(dict_df_score_fracs, frac=0.99, n_cols=5, n_rows=None, plot_type='violin'):\n",
    "    if n_rows is None:\n",
    "        n_rows = int(math.ceil(len(dict_df_score_fracs) / n_cols))\n",
    "        \n",
    "    fig, axs = plt.subplots(n_rows, n_cols, figsize=(n_cols * 4, n_rows * 2.85))\n",
    "\n",
    "\n",
    "    for idx, (key, val) in enumerate(dict_df_score_fracs.items()):\n",
    "        val = val.sort_values(by='cluster')\n",
    "        val = val[~ val['cluster'].isin(['T1', 'U'])]\n",
    "        if plot_type == 'violin':\n",
    "            v = sns.violinplot(data=val, x='cluster', y=str(frac), scale='width', cut=0, inner=None,\n",
    "                           ax = axs.ravel()[idx], linewidth=1,\n",
    "                   palette=[dict_colors_human_mouse[h] for  h in sorted(set(val['cluster'].values))]\n",
    "                          ).set(xlabel=None, ylabel=None, yticks=[], title=' '.join(key.split('_')[:2]))\n",
    "            vb = sns.boxplot(data=val,  x='cluster', y=str(frac), width=0.2, fliersize=0, ax = axs.ravel()[idx], \n",
    "            linewidth=0, showcaps=False, boxprops={'zorder': 2}, color=\"#232323\", \n",
    "            whiskerprops=dict(color=\"#232323\", linewidth=1.8),\n",
    "            medianprops=dict(color=\"#ffffff\", linewidth=6, linestyle=':'))\n",
    "        elif plot_type == 'boxplot':\n",
    "            b = sns.boxplot(data=val, x='cluster', y=str(frac), fliersize=0,\n",
    "            ax = axs.ravel()[idx], linewidth=1,\n",
    "                   palette=[dict_colors_human_mouse[h] for  h in sorted(set(val['cluster'].values))]\n",
    "                          ).set(xlabel=None, ylabel=None, yticks=[], title=' '.join(key.split('_')[:2]))\n",
    "            \n",
    "        _ = axs.ravel()[idx].set_xticklabels(sorted(set(val['cluster'].values)), size='large')\n",
    "        _ = axs.ravel()[idx].set_yticks([0, 1])\n",
    "        _ = axs.ravel()[idx].set_yticklabels([0, 1], size='medium')\n",
    "        \n",
    "        sns.despine(left=True, bottom=True)\n",
    "        \n",
    "    for idx in range(len(dict_df_score_fracs), n_cols*n_rows):\n",
    "        axs.ravel()[idx].set_axis_off()\n",
    "        \n",
    "    plt.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "841e32c6-02c4-4274-b745-13e1a83e20d2",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for plot_type in ['violin', 'boxplot']:\n",
    "    plot_distributions_bootstrap(dict_df_score_fracs_human, frac=0.99, n_cols=4, n_rows=None, plot_type=plot_type)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1da08c32-9fba-4b75-952d-fdb6b3ab31c9",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for plot_type in ['violin', 'boxplot']:\n",
    "    plot_distributions_bootstrap(dict_df_score_fracs_mouse, frac=0.99, n_cols=3, n_rows=None, plot_type=plot_type)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e6f6048a-9886-4557-b258-8620cc168599",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def plot_distributions_bootstrap_dataset(dict_df_score_fracs, frac=0.99, n_cols=5, n_rows=None, plot_type='violin'):\n",
    "    list_dfs = []\n",
    "    for key, val in dict_df_score_fracs.items():\n",
    "        name = ' '.join(key.split('_')[:2])\n",
    "        val['name'] = name\n",
    "        list_dfs.append(val[['name', 'cluster', f\"{frac}\"]])\n",
    "    \n",
    "    df_joint = pd.concat(list_dfs, axis=0, ignore_index=True)\n",
    "\n",
    "\n",
    "    list_clusters = sorted(set(df_joint['cluster'].values))\n",
    "    list_clusters = [i for i in list_clusters if i not in ['T1', 'U']]\n",
    "    \n",
    "    if n_rows is None:\n",
    "        n_rows = int(math.ceil(len(list_clusters) / n_cols))\n",
    "        \n",
    "    fig, axs = plt.subplots(n_rows, n_cols, figsize=(n_cols * 4, n_rows * 2.85))\n",
    "\n",
    "\n",
    "    for idx, cluster in enumerate(list_clusters):\n",
    "        df_joint_cluster = df_joint[df_joint['cluster'] == cluster].sort_values(by='name') \n",
    "        \n",
    "        if plot_type == 'violin':\n",
    "            v = sns.violinplot(data=df_joint_cluster, x='name', y=str(frac), scale='width', cut=0, inner='box',\n",
    "                           ax = axs.ravel()[idx], linewidth=1,\n",
    "                   color=dict_colors_human_mouse[cluster],\n",
    "                          ).set(xlabel=None, ylabel=None, yticks=[], title=cluster)\n",
    "            \n",
    "        elif plot_type == 'boxplot':\n",
    "            b = sns.boxplot(data=df_joint_cluster, x='name', y=str(frac), fliersize=0,\n",
    "            ax = axs.ravel()[idx], linewidth=1,\n",
    "                   color=dict_colors_human_mouse[cluster],\n",
    "                          ).set(xlabel=None, ylabel=None, yticks=[], title=cluster)\n",
    "            \n",
    "\n",
    "        _ = axs.ravel()[idx].set_xticklabels(sorted(set(df_joint_cluster['name'].values)), \n",
    "                                                 size='small', rotation=45, ha='right')\n",
    "        _ = axs.ravel()[idx].set_yticks([0, 1])\n",
    "        _ = axs.ravel()[idx].set_yticklabels([0, 1], size='medium')\n",
    "        \n",
    "        sns.despine(left=True, bottom=True)\n",
    "        \n",
    "    for idx in range(len(list_clusters), n_cols*n_rows):\n",
    "        axs.ravel()[idx].set_axis_off()\n",
    "        \n",
    "    plt.tight_layout()"
   ]
  },
  {
   "cell_type": "raw",
   "id": "38350af8-966f-4337-9b0e-8e353a3a3bbc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-29T15:37:31.644185Z",
     "iopub.status.busy": "2022-12-29T15:37:31.643981Z",
     "iopub.status.idle": "2022-12-29T15:37:40.782314Z",
     "shell.execute_reply": "2022-12-29T15:37:40.781459Z",
     "shell.execute_reply.started": "2022-12-29T15:37:31.644159Z"
    },
    "tags": []
   },
   "source": [
    "plot_distributions_bootstrap_dataset(dict_df_score_fracs_human, frac=0.99, n_cols=3, n_rows=None, \n",
    "                                     plot_type='boxplot')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "83bda12d-dee6-49f7-a847-dd0ec3445f76",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "213d17d9-75a4-492f-8abf-9e1962a72dd8",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot_distributions_bootstrap_dataset(dict_df_score_fracs_mouse, frac=0.99, n_cols=4, n_rows=None, \n",
    "                                     plot_type='boxplot')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f64e5abc-ec09-43df-b5af-421deb8d1a58",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def plot_distributions_bootstrap_general(dict_df_score_fracs, frac=0.99, n_cols=5, n_rows=None, plot_type='violin'):\n",
    "    list_dfs = []\n",
    "    for key, val in dict_df_score_fracs.items():\n",
    "        name = ' '.join(key.split('_')[:2])\n",
    "        val['name'] = name\n",
    "        list_dfs.append(val[['name', 'cluster', f\"{frac}\"]])\n",
    "    \n",
    "    df_joint = pd.concat(list_dfs, axis=0, ignore_index=True)\n",
    "    df_joint = df_joint[~ df_joint['cluster'].isin(['T1', 'U'])]\n",
    "    df_joint = df_joint.groupby(['name', 'cluster']).mean().reset_index()\n",
    "    \n",
    "#     return df_joint\n",
    "    \n",
    "    fig, axs = plt.subplots(1, 2, figsize=(2 * 4, 1 * 2.85))\n",
    "        \n",
    "    if plot_type == 'violin':\n",
    "        v1 = sns.violinplot(data=df_joint, x='name', y=str(frac), scale='width', cut=0, inner='box',\n",
    "            ax = axs[0], linewidth=1,\n",
    "               color=\"#bcbcbc\").set(xlabel=None, ylabel=None, yticks=[], title='Dataset')\n",
    "        v2 = sns.violinplot(data=df_joint, x='cluster', y=str(frac), scale='width', cut=0, inner='box',\n",
    "            ax = axs[1], linewidth=1,\n",
    "               palette=[dict_colors_human_mouse[h] for  h in sorted(set(df_joint['cluster'].values))],\n",
    "                      ).set(xlabel=None, ylabel=None, yticks=[], title='Cluster')\n",
    "        \n",
    "        \n",
    "    elif plot_type == 'boxplot':\n",
    "        b1 = sns.boxplot(data=df_joint, x='name', y=str(frac), fliersize=0,\n",
    "            ax = axs[0], linewidth=1,\n",
    "                   color=\"#bcbcbc\",\n",
    "                          ).set(xlabel=None, ylabel=None, yticks=[], title='Dataset')\n",
    "        \n",
    "        b2 = sns.boxplot(data=df_joint, x='cluster', y=str(frac), fliersize=0,\n",
    "            ax = axs[1], linewidth=1,\n",
    "                   palette=[dict_colors_human_mouse[h] for  h in sorted(set(df_joint['cluster'].values))],\n",
    "                          ).set(xlabel=None, ylabel=None, yticks=[], title='Cluster')\n",
    "\n",
    "    \n",
    "    _ = axs[0].set_xticklabels(sorted(set(df_joint['name'].values)), \n",
    "                                                 size='small', rotation=45, ha='right')\n",
    "    _ = axs[1].set_xticklabels(sorted(set(df_joint['cluster'].values)), \n",
    "                                                 size='medium')\n",
    "    \n",
    "    for idx in [0, 1]:\n",
    "        _ = axs[idx].set_yticks([0, 1])\n",
    "        _ = axs[idx].set_yticklabels([0, 1], size='medium')\n",
    "        \n",
    "    sns.despine(left=True, bottom=True)\n",
    "                \n",
    "    plt.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "27a00c5c-65ab-4c3b-97eb-0393c6b33f23",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot_distributions_bootstrap_general(dict_df_score_fracs_human, frac=0.99, n_cols=2, n_rows=None, plot_type=plot_type)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0bd0601a-f34d-46c3-9ffb-155375ac5e69",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot_distributions_bootstrap_general(dict_df_score_fracs_mouse, frac=0.99, n_cols=2, n_rows=None, plot_type=plot_type)"
   ]
  },
  {
   "cell_type": "raw",
   "id": "9ea7b152-e078-4a44-adfb-1927b5890c61",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-16T17:30:56.123780Z",
     "iopub.status.busy": "2022-11-16T17:30:56.123274Z",
     "iopub.status.idle": "2022-11-16T17:30:56.159383Z",
     "shell.execute_reply": "2022-11-16T17:30:56.158208Z",
     "shell.execute_reply.started": "2022-11-16T17:30:56.123723Z"
    },
    "tags": []
   },
   "source": [
    "## Calculating the probability of a cell type being assigned to the same, or other cell types"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1496e764-2e3a-475d-aa7d-9d0e052a7174",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def assign_cluster_score(df_bootstrap, N_iters):\n",
    "    df_flat = df_bootstrap.loc[:, [str(i) for i in range(N_iters)]].values.ravel()\n",
    "    cell_types_out = sorted(set(df_flat[~ pd.isnull(df_flat)]))    \n",
    "    cell_types_in = sorted(set(df_bootstrap.loc[:, '-1'].values.ravel()))\n",
    "    \n",
    "    df_adjacency = pd.DataFrame(0, index=cell_types_in, columns=cell_types_out)\n",
    "    \n",
    "    for cell_type_in in cell_types_in:\n",
    "        list_vals = []\n",
    "        df_sub = df_bootstrap[df_bootstrap['-1'] == cell_type_in].loc[:, [str(i) for i in range(N_iters)]]\n",
    "        vals_Na = df_sub.values.ravel()\n",
    "        vals = vals_Na[~ pd.isnull(vals_Na)]\n",
    "        \n",
    "        for cell_type_out in cell_types_out:\n",
    "            list_vals.append(np.sum(vals == cell_type_out) / len(vals))\n",
    "        \n",
    "        df_adjacency.loc[cell_type_in, :] = list_vals\n",
    "    \n",
    "    return df_adjacency"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "06b99737-0bc0-4f43-832d-058275619569",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dict_adjacency_dfs = {}\n",
    "for key, df_bootstrap in dict_bootstraps.items():\n",
    "    adjacency_df = assign_cluster_score(df_bootstrap, N_iters)\n",
    "    dict_adjacency_dfs[key] = adjacency_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "45cb81ff-e115-4b1c-9009-1fddbe27e2ee",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def plot_distributions_bootstrap(dict_adjacency_dfs, n_cols=5, n_rows=None):\n",
    "    if n_rows is None:\n",
    "        n_rows = int(math.ceil(len(dict_adjacency_dfs) / n_cols))\n",
    "        \n",
    "    fig, axs = plt.subplots(n_rows, n_cols, figsize=(n_cols * 3, n_rows * 3))\n",
    "\n",
    "\n",
    "    for idx, (key, val) in enumerate(dict_adjacency_dfs.items()):\n",
    "        val_nan = val.copy()\n",
    "        val_nan = val_nan.loc[[i for i in val_nan.index if i not in ['T1', 'U']], [i for i in val_nan.columns if i not in ['T1', 'U']]]\n",
    "        val_nan[val_nan < 0.01] = np.NaN\n",
    "        sns.heatmap(round_df(val_nan), annot=True, cmap='Blues', annot_kws={\"fontsize\": 'x-small'}, fmt='.2f', yticklabels=True, \n",
    "                    xticklabels=True, cbar=False, ax=axs.ravel()[idx], square=True)\n",
    "        [t.set_color(dict_colors_human_mouse[t.get_text()]) for t in axs.ravel()[idx].xaxis.get_ticklabels()] \n",
    "        [t.set_color(dict_colors_human_mouse[t.get_text()]) for t in axs.ravel()[idx].yaxis.get_ticklabels()]\n",
    "        axs.ravel()[idx].set_xticklabels(axs.ravel()[idx].get_xticklabels(), size='medium', weight='bold',)\n",
    "        axs.ravel()[idx].set_yticklabels(axs.ravel()[idx].get_yticklabels(), va='center', size='medium', weight='bold')\n",
    "        axs.ravel()[idx].set_title(' '.join(key.split('_')[:2]))\n",
    "        \n",
    "        change_texts_format(axs.ravel()[idx])\n",
    "        \n",
    "    for idx in range(len(dict_df_score_fracs), n_cols*n_rows):\n",
    "        axs.ravel()[idx].set_axis_off()\n",
    "        \n",
    "\n",
    "        \n",
    "    plt.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5b35e73b-4251-4675-9a14-182140154162",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dict_adjacency_dfs_human_099 = {key: dict_adjacency_dfs[key] for key in sorted([key for key in dict_adjacency_dfs.keys() if ('human' in key) & ('0.99' in key)])}\n",
    "plot_distributions_bootstrap(dict_adjacency_dfs_human_099)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee93560e-f83e-43d5-a200-2a58a6fecb23",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dict_adjacency_dfs_mouse_099 = {key: dict_adjacency_dfs[key] for key in sorted([key for key in dict_adjacency_dfs.keys() if ('mouse' in key) & ('0.99' in key)])}\n",
    "plot_distributions_bootstrap(dict_adjacency_dfs_mouse_099, n_cols=3, )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5ad8c88d-bb1d-4ee6-8d9c-2cc06e24c330",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Unified plot\n",
    "def make_unified_adjacency(list_adjacency_dfs):\n",
    "    \"\"\"\n",
    "    This unified adjacency matrix is calculated by obtaining the median values of all dfs. \n",
    "    However, since the individual dataframes do not have all clusters, we need to apply a nanmean of all the values.\n",
    "    So, we are going to go pair by pair, get the values of all dfs, and add NaNs to the ones that do not exist, and\n",
    "    calculate the nanmean value.\n",
    "    Then, we are going to normalize the probabilities rowwise to sum 1.\n",
    "    \"\"\"\n",
    "    \n",
    "    list_clusters = []\n",
    "    for df in list_adjacency_dfs:\n",
    "        list_clusters += df.index.tolist()\n",
    "    \n",
    "    list_clusters = sorted(set([i for i in list_clusters if i not in ['U', 'T1']]))\n",
    "    \n",
    "    df_unified = pd.DataFrame(index=list_clusters, columns=list_clusters)\n",
    "    \n",
    "    for c_i in list_clusters:\n",
    "        for c_o in list_clusters:\n",
    "            list_vals = []\n",
    "            for df in list_adjacency_dfs:\n",
    "                if (c_i in df.index) & (c_o in df.columns):\n",
    "                    list_vals.append(df.loc[c_i, c_o])\n",
    "                \n",
    "            df_unified.loc[c_i, c_o] = np.median(list_vals)\n",
    "    \n",
    "    df_unified = df_unified.astype(float)\n",
    "    df_unified = df_unified.div(df_unified.sum(axis=1), axis=0)\n",
    "   \n",
    "    return df_unified"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee52db87-b5ed-4fda-957f-6fe676e66b19",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_unified_human = make_unified_adjacency([val for (key, val) in dict_adjacency_dfs.items() if ('0.99' in key) & ('human' in key)])\n",
    "df_unified_human[df_unified_human < 0.01] = np.NaN\n",
    "\n",
    "fig, ax = plt.subplots(1, 1)\n",
    "sns.heatmap(round_df(df_unified_human), annot=True, cmap='Blues', annot_kws={\"fontsize\": 'x-small'}, yticklabels=True, cbar=False)\n",
    "[t.set_color(dict_colors_human[t.get_text()]) for t in ax.xaxis.get_ticklabels()]; [t.set_color(dict_colors_human[t.get_text()]) for t in ax.yaxis.get_ticklabels()]\n",
    "ax.set_xticklabels(ax.get_xticklabels(),  weight='bold'); ax.set_yticklabels(ax.get_yticklabels(),  weight='bold', va='center')\n",
    "change_texts_format(ax)\n",
    "None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e715f268-d673-4009-8056-6bfd4eff099b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_unified_mouse = make_unified_adjacency([val for (key, val) in dict_adjacency_dfs.items() if ('0.99' in key) & ('mouse' in key)])\n",
    "df_unified_mouse[df_unified_mouse < 0.01] = np.NaN\n",
    "\n",
    "fig, ax = plt.subplots(1, 1, figsize=(7, 5))\n",
    "sns.heatmap(round_df(df_unified_mouse), annot=True, cmap='Blues', annot_kws={\"fontsize\": 'x-small'}, yticklabels=True, cbar=False)\n",
    "[t.set_color(dict_colors_mouse[t.get_text()]) for t in ax.xaxis.get_ticklabels()]; [t.set_color(dict_colors_mouse[t.get_text()]) for t in ax.yaxis.get_ticklabels()]\n",
    "ax.set_xticklabels(ax.get_xticklabels(),  weight='bold'); ax.set_yticklabels(ax.get_yticklabels(),  weight='bold', va='center')\n",
    "None"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "17d8d8c8-9313-4b21-8414-9f4495192aae",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-16T17:30:56.123780Z",
     "iopub.status.busy": "2022-11-16T17:30:56.123274Z",
     "iopub.status.idle": "2022-11-16T17:30:56.159383Z",
     "shell.execute_reply": "2022-11-16T17:30:56.158208Z",
     "shell.execute_reply.started": "2022-11-16T17:30:56.123723Z"
    },
    "tags": []
   },
   "source": [
    "## Calculate the probability of a cell type having a higher cluster score than a certain value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "33aea114-0d74-474b-a10d-d69a8018b861",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def calculate_proportion_cell_higher_prop(df_bootstrap, prop):   \n",
    "    a, b = np.unique(df_bootstrap['-1'], return_counts=True)\n",
    "    dict_all = dict(zip(a.tolist(), b.tolist()))\n",
    "\n",
    "    a, b = np.unique(df_bootstrap[df_bootstrap['same_cluster_prop'] > prop]['-1'], return_counts=True)\n",
    "    dict_short = dict(zip(a.tolist(), b.tolist()))\n",
    "\n",
    "    dict_prop = dict((key, dict_short.get(key, 0)/val) for key, val in dict_all.items())\n",
    "    \n",
    "    return dict_prop\n",
    "\n",
    "# def calculate_proportion_cell_higher_prop(df_bootstrap, prop):   \n",
    "#     a, b = np.unique(df_bootstrap['-1'], return_counts=True)\n",
    "    \n",
    "    \n",
    "#     dict_prop = {i: df_bootstrap[df_bootstrap['-1'] == i]['same_cluster_prop'].mean() for i in a}\n",
    "    \n",
    "#     return dict_prop\n",
    "\n",
    "def calculate_df_proportion_cell_higher_prop(dict_bootstraps, prop):\n",
    "    list_clusters = []\n",
    "    for df in dict_bootstraps.values():\n",
    "        list_clusters += df['-1'].values.tolist()\n",
    "    \n",
    "    list_clusters = sorted(set([i for i in list_clusters]))\n",
    "    \n",
    "    df_return = pd.DataFrame(columns=list_clusters, index=[i for i in dict_bootstraps.keys()])\n",
    "    \n",
    "    \n",
    "    for name, df_bootstrap in dict_bootstraps.items():\n",
    "        dict_prop = calculate_proportion_cell_higher_prop(df_bootstrap, prop)        \n",
    "        df_return.loc[name, list(dict_prop.keys())] = list(dict_prop.values())\n",
    "    \n",
    "    return df_return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "47abca44-c9b5-4517-bb5c-63ba6fcecbe5",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dict_human_099 = {key: dict_bootstraps[key] for key in sorted([key for key in dict_bootstraps.keys() if ('human' in key) & ('0.99' in key)])}\n",
    "df_prop = calculate_df_proportion_cell_higher_prop(dict_human_099, 0.7)\n",
    "df_prop = df_prop[[col for col in df_prop.columns if col not in ['T1', 'U']]]\n",
    "df_prop = df_prop.astype(float)\n",
    "df_prop = df_prop.loc[df_prop.mean(1).sort_values().index[::-1]]\n",
    "\n",
    "df_prop['mean (dataset)'] = df_prop.mean(1)\n",
    "df_prop.loc['mean (cluster)'] = df_prop.mean(0)\n",
    "df_prop.iloc[-1, -1] = np.NaN\n",
    "\n",
    "fig, ax = plt.subplots(1, 1, figsize=(6, 5))\n",
    "g = sns.heatmap(df_prop, annot=True, cmap='Blues', yticklabels=True, annot_kws={\"fontsize\": 'xx-small'}, \n",
    "                fmt='.2f', cbar=False)\n",
    "g.set_yticklabels([' '.join(i.get_text().split('_')[:2])  for i in g.get_ymajorticklabels()], fontsize = 'x-small')\n",
    "g.set_xticklabels(g.get_xmajorticklabels(), fontsize = 'large')\n",
    "[t.set_color(dict_colors_human[t.get_text()]) if t.get_text() in dict_colors_human else t.set_color(\"#000000\") for t in ax.xaxis.get_ticklabels() ]\n",
    "[t.set_color(dict_colors_human[t.get_text()]) if t.get_text() in dict_colors_human else t.set_color(\"#000000\") for t in ax.yaxis.get_ticklabels()]\n",
    "ax.set_xticklabels(ax.get_xticklabels(),  weight='bold'); \n",
    "change_texts_format(ax)\n",
    "None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "56f79684-97c7-47aa-9f48-6276aa01ee4a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dict_mouse_099 = {key: dict_bootstraps[key] for key in sorted([key for key in dict_bootstraps.keys() if ('mouse' in key) & ('0.99' in key)])}\n",
    "df_prop = calculate_df_proportion_cell_higher_prop(dict_mouse_099, 0.7)\n",
    "df_prop = df_prop[[col for col in df_prop.columns if col not in ['T1', 'U']]]\n",
    "df_prop = df_prop.astype(float)\n",
    "df_prop = df_prop.loc[df_prop.mean(1).sort_values().index[::-1]]\n",
    "\n",
    "df_prop['mean (dataset)'] = df_prop.mean(1)\n",
    "df_prop.loc['mean (cluster)'] = df_prop.mean(0)\n",
    "df_prop.iloc[-1, -1] = np.NaN\n",
    "\n",
    "fig, ax = plt.subplots(1, 1, figsize=(6, 2))\n",
    "g = sns.heatmap(df_prop, annot=True, cmap='Blues', yticklabels=True, annot_kws={\"fontsize\": 'xx-small'}, \n",
    "                fmt='.2f', cbar=False)\n",
    "g.set_yticklabels([' '.join(i.get_text().split('_')[:2])  for i in g.get_ymajorticklabels()], fontsize = 'x-small')\n",
    "g.set_xticklabels(g.get_xmajorticklabels(), fontsize = 'large')\n",
    "[t.set_color(dict_colors_mouse[t.get_text()]) if t.get_text() in dict_colors_mouse else t.set_color(\"#000000\") for t in ax.xaxis.get_ticklabels() ]\n",
    "[t.set_color(dict_colors_mouse[t.get_text()]) if t.get_text() in dict_colors_mouse else t.set_color(\"#000000\") for t in ax.yaxis.get_ticklabels()]\n",
    "ax.set_xticklabels(ax.get_xticklabels(),  weight='bold'); \n",
    "change_texts_format(ax)\n",
    "None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1a410d53-474f-4665-9a06-78e076aa78fe",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7c9819f2-483f-4a17-8bfd-75ff4181340c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "088c6ff1-74c4-42db-961f-cb97d59c0333",
   "metadata": {},
   "outputs": [],
   "source": [
    "os.system('jupyter nbconvert --to html 10HM_Robustness_tests.ipynb')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a7de20b-975c-48b3-a4fc-2682c778e405",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:triku-notebooks]",
   "language": "python",
   "name": "conda-env-triku-notebooks-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}