4H_analysis_of_markers_and_pathways.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Obtaining robust cell population markers, and redefining/reassuring the biased cell populations\n",
    "\n",
    "**TO RUN THIS NOTEBOOK YOU HAVE TO RUN 3H NOTEBOOK FULLY!!!**\n",
    "\n",
    "For part of the analysis **you need to run the 0A notbeook** too. This part is not strictly necessary to be run, but instead it helps me do part of some gene analyses quicker. If you have trouble loading content from this notebook, ignore it because the analysis will be the same."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import scanpy as sc\n",
    "import scanpy.external as sce\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import triku as tk\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib as mpl\n",
    "from tqdm.notebook import tqdm\n",
    "import scipy.sparse as spr\n",
    "import networkx as nx\n",
    "from kneed import KneeLocator\n",
    "from matplotlib import pylab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!pip install cellassign"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# FROM NOTEBOOK 0A!!\n",
    "%store -r df_human_genes_codes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# local imports and imports from other notebooks\n",
    "from cellassign import assign_cats\n",
    "from fb_functions import make_gene_scoring_with_expr, plot_score_graph, plot_UMAPS_gene, plot_adata_cluster_properties, make_dicts_fraction_mean, plot_dotplot_gene, plot_dotplot_list_genes, make_combined_table\n",
    "%store -r dict_colors_human\n",
    "%store -r seed\n",
    "%store -r magma\n",
    "%store -r data_dir\n",
    "%store -r dict_cats_fb_human"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%store -r plot_params\n",
    "\n",
    "pylab.rcParams.update(plot_params)\n",
    "pd.set_option('display.max_columns', None)\n",
    "pd.options.display.float_format = \"{:,.2f}\".format"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**IMPORTANT: I am running this analysis in a computer with ~500 GB of RAM. I will load many datasets at once, which might be too much for some computers. I took this decision conciously, to have as much info available at any time as possible. If you cannot run all the analysis at once, you can run it by parts.**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Anndata loading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "ahlers_2022_dir = data_dir + '/ahlers_2022'\n",
    "ahlers_2022_young_human_fb = sc.read(ahlers_2022_dir + '/ahlers_2022_young_human_fb_robust.h5')\n",
    "ahlers_2022_young_human = sc.read(ahlers_2022_dir + '/ahlers_2022_young_human_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "billi_2022_dir = data_dir + '/billi_2022'\n",
    "billi_2022_ctrl_human_fb = sc.read(billi_2022_dir + '/billi_2022_ctrl_human_fb_processed.h5')\n",
    "billi_2022_ctrl_human = sc.read(billi_2022_dir + '/billi_2022_ctrl_human_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "boothby_2021_dir = data_dir + '/boothby_2021'\n",
    "boothby_2021_ctrl_human_fb = sc.read(boothby_2021_dir + '/boothby_2021_ctrl_human_fb_robust.h5')\n",
    "boothby_2021_ctrl_human = sc.read(boothby_2021_dir + '/boothby_2021_ctrl_human_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "burja_2022_dir = data_dir + '/burja_2022'\n",
    "burja_2022_ctrl_human_fb = sc.read(burja_2022_dir + '/burja_2022_ctrl_human_fb_robust.h5')\n",
    "burja_2022_ctrl_human = sc.read(burja_2022_dir + '/burja_2022_ctrl_human_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "deng_2021_dir = data_dir + '/deng_2021'\n",
    "deng_2021_scar_fb = sc.read(deng_2021_dir + '/deng_2021_scar_fb_robust.h5')\n",
    "deng_2021_scar = sc.read(deng_2021_dir + '/deng_2021_scar_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "gao_2021_dir = data_dir + '/gao_2021'\n",
    "gao_2021_ctrl_human_fb = sc.read(gao_2021_dir + '/gao_2021_ctrl_human_fb_robust.h5')\n",
    "gao_2021_ctrl_human = sc.read(gao_2021_dir + '/gao_2021_ctrl_human_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "gaydosik_2020_dir = data_dir + '/gaydosik_2020'\n",
    "gaydosik_2020_ctrl_human_fb = sc.read(gaydosik_2020_dir + '/gaydosik_2020_ctrl_human_fb_robust.h5')\n",
    "gaydosik_2020_ctrl_human = sc.read(gaydosik_2020_dir + '/gaydosik_2020_ctrl_human_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "gur_2022_dir = data_dir + '/gur_2022'\n",
    "gur_2022_ctrl_human_fb = sc.read(gur_2022_dir + '/gur_2022_ctrl_human_fb_robust.h5')\n",
    "gur_2022_ctrl_human = sc.read(gur_2022_dir + '/gur_2022_ctrl_human_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "he_2020_dir = data_dir + '/He_2020'\n",
    "he_2020_ctrl_human_fb = sc.read(he_2020_dir + '/he_2020_ctrl_human_fb_robust.h5')\n",
    "he_2020_ctrl_human = sc.read(he_2020_dir + '/he_2020_ctrl_human_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "hughes_2020_dir = data_dir + '/hughes_2020'\n",
    "hughes_2020_ctrl_human_fb = sc.read(hughes_2020_dir + '/hughes_2020_ctrl_human_fb_robust.h5')\n",
    "hughes_2020_ctrl_human = sc.read(hughes_2020_dir + '/hughes_2020_ctrl_human_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "kim_2020_dir = data_dir + '/Kim_2020'\n",
    "kim_2020_ctrl_human_fb = sc.read(kim_2020_dir + '/kim_2020_ctrl_human_fb_robust.h5')\n",
    "kim_2020_ctrl_human = sc.read(kim_2020_dir + '/kim_2020_ctrl_human_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "liu_2021_dir = data_dir + '/liu_2021'\n",
    "liu_2021_ctrl_human_fb = sc.read(liu_2021_dir + '/liu_2021_ctrl_human_fb_robust.h5')\n",
    "liu_2021_ctrl_human = sc.read(liu_2021_dir + '/liu_2021_ctrl_human_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "mariottoni_2021_dir = data_dir + '/mariottoni_2021'\n",
    "mariottoni_2021_ctrl_human_fb = sc.read(mariottoni_2021_dir + '/mariottoni_2021_ctrl_human_fb_robust.h5')\n",
    "mariottoni_2021_ctrl_human = sc.read(mariottoni_2021_dir + '/mariottoni_2021_ctrl_human_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "mirizio_2020_dir = data_dir + '/mirizio_2020'\n",
    "mirizio_2020_scl_human_fb = sc.read(mirizio_2020_dir + '/mirizio_2020_scl_human_fb_robust.h5')\n",
    "mirizio_2020_scl_human = sc.read(mirizio_2020_dir + '/mirizio_2020_scl_human_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "reynolds_2021_dir = data_dir + '/reynolds_2021'\n",
    "reynolds_2021_ctrl_human_fb = sc.read(reynolds_2021_dir + '/reynolds_2021_ctrl_human_fb_robust.h5')\n",
    "# Dataset with the rest of cells is not available. It can be used but since the dataset doesn't have good quality, is not fully relevant."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "rindler_2021_dir = data_dir + '/rindler_2021'\n",
    "rindler_2021_ctrl_human_fb = sc.read(rindler_2021_dir + '/rindler_2021_ctrl_human_fb_robust.h5')\n",
    "rindler_2021_ctrl_human = sc.read(rindler_2021_dir + '/rindler_2021_ctrl_human_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sole_2020_dir = data_dir + '/Sole-Boldo_2020'\n",
    "sole_2020_young_human_fb = sc.read(sole_2020_dir + '/sole_2020_young_human_fb_robust.h5')\n",
    "sole_2020_young_human = sc.read(sole_2020_dir + '/sole_2020_young_human_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "tabib_2018_dir = data_dir + '/Tabib_2018'\n",
    "tabib_2018_ctrl_human_fb = sc.read(tabib_2018_dir + '/tabib_2018_ctrl_human_fb_robust.h5')\n",
    "tabib_2018_ctrl_human = sc.read(tabib_2018_dir + '/tabib_2018_ctrl_human_processed.h5')\n",
    "\n",
    "# By default in the analysis the characterization of fb was already done, and the \n",
    "# UMAP was not computed\n",
    "sc.pp.log1p(tabib_2018_ctrl_human)\n",
    "sc.pp.pca(tabib_2018_ctrl_human)\n",
    "sc.pp.neighbors(tabib_2018_ctrl_human)\n",
    "sc.tl.umap(tabib_2018_ctrl_human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "tabib_2021_dir = data_dir + '/Tabib_2021'\n",
    "tabib_2021_ctrl_human_fb = sc.read(tabib_2021_dir + '/tabib_2021_ctrl_human_fb_robust.h5')\n",
    "tabib_2021_ctrl_human = sc.read(tabib_2021_dir + '/tabib_2021_ctrl_human_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "theo_2020_dir = data_dir + '/Theocharidis_2020/'\n",
    "theo_2020_ctrl_human_dm_fb = sc.read(theo_2020_dir + '/theo_2020_ctrl_human_dm_fb_robust.h5')\n",
    "theo_2020_ctrl_human_dm = sc.read(theo_2020_dir + '/theo_2020_ctrl_human_dm_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "theo_2021_dir = data_dir + '/Theocharidis_2021/'\n",
    "theo_2021_ctrl_human_fb = sc.read(theo_2021_dir + '/theo_2021_ctrl_human_fb_robust.h5')\n",
    "theo_2021_ctrl_human = sc.read(theo_2021_dir + '/theo_2021_ctrl_human_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "tsc_2021_dir = data_dir + '/Tabula_Sapiens_Consortium_2021'\n",
    "tsc_2021_ctrl_human_fb = sc.read(tsc_2021_dir + '/tsc_2021_ctrl_human_processed.h5')\n",
    "tsc_2021_ctrl_human = sc.read(tsc_2021_dir + '/tsc_2021_ctrl_human_fb_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "vors_2020_dir = data_dir + '/Vorstandlechner_2020'\n",
    "vors_2020_ctrl_human_fb = sc.read(vors_2020_dir + '/vors_2020_ctrl_human_fb_robust.h5')\n",
    "vors_2020_ctrl_human = sc.read(vors_2020_dir + '/vors_2020_ctrl_human_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "vors_2021_dir = data_dir + '/Vorstandlechner_2021'\n",
    "vors_2021_ctrl_human_fb = sc.read(vors_2021_dir + '/vors_2021_ctrl_human_fb_robust.h5')\n",
    "vors_2021_ctrl_human = sc.read(vors_2021_dir + '/vors_2021_ctrl_human_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "xu_2021_dir = data_dir + '/xu_2021'\n",
    "xu_2021_ctrl_human_fb = sc.read(xu_2021_dir + '/xu_2021_ctrl_human_fb_robust.h5')\n",
    "xu_2021_ctrl_human = sc.read(xu_2021_dir + '/xu_2021_ctrl_human_processed.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Geting the definitive list of genes\n",
    "This is not *really* definitive. We use the produced markers to later on, in the analysis table (not in notebooks) do a selection based on gene function, or pattern of expression. The *final* list is in the variable **genes** below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%store -r list_all_datasets_human\n",
    "%store -r list_datasets_human\n",
    "\n",
    "%store -r list_accepted_clusters_human\n",
    "%store -r list_accepted_axis_human\n",
    "%store -r list_names_human"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "dict_make_gene_scoring_cluster_robust_human = make_gene_scoring_with_expr(list_datasets=list_datasets_human, calculate_DEGs = True, group_name = 'cluster_robust', \n",
    "                                                     value_ref = 'scores', select_method = 'pval', \n",
    "                                                     list_clusters = list_accepted_clusters_human,\n",
    "                                                     select_thres = 0.05)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dict_make_gene_scoring_axis_robust_human = make_gene_scoring_with_expr(list_datasets=list_datasets_human, calculate_DEGs = True, group_name = 'axis_robust',\n",
    "                                                     value_ref = 'scores', select_method = 'pval', select_thres = 0.05, \n",
    "                                                         list_clusters =list_accepted_axis_human,)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%store dict_make_gene_scoring_cluster_robust_human\n",
    "%store dict_make_gene_scoring_axis_robust_human"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dict_make_gene_scoring_cluster_robust_human['E1'].iloc[:50].sort_index().index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dict_make_gene_scoring_cluster_robust_human['A1'].iloc[:50].sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dict_make_gene_scoring_cluster_robust_human['A2'].iloc[:50].sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dict_make_gene_scoring_cluster_robust_human['A3'].iloc[:50].sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dict_make_gene_scoring_cluster_robust_human['A4'].iloc[:50].sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dict_make_gene_scoring_cluster_robust_human['B1'].iloc[:50].sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dict_make_gene_scoring_cluster_robust_human['B2'].iloc[:50].sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dict_make_gene_scoring_cluster_robust_human['B3'].iloc[:50].sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dict_make_gene_scoring_cluster_robust_human['B4'].iloc[:50].sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dict_make_gene_scoring_cluster_robust_human['C1'].iloc[:50].sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dict_make_gene_scoring_cluster_robust_human['C2'].iloc[:50].sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dict_make_gene_scoring_cluster_robust_human['C3'].iloc[:50].sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dict_make_gene_scoring_cluster_robust_human['C5'].iloc[:50].sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dict_make_gene_scoring_cluster_robust_human['D1'].iloc[:50].sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dict_make_gene_scoring_cluster_robust_human['D2'].iloc[:50].sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dict_make_gene_scoring_cluster_robust_human['D2'].iloc[:50].sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dict_make_gene_scoring_cluster_robust_human['E1'].iloc[:50].sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dict_make_gene_scoring_cluster_robust_human['T1'].iloc[:50].sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# This variable is useful for the Ligand-Receptor notebook, look all the details there\n",
    "dict_knee_markers = {}\n",
    "\n",
    "for clus in dict_make_gene_scoring_cluster_robust_human.keys():\n",
    "    kneedle = KneeLocator(range(len(dict_make_gene_scoring_cluster_robust_human[clus])), dict_make_gene_scoring_cluster_robust_human[clus].Z, \n",
    "                          S=1.0, curve=\"convex\", direction=\"decreasing\")\n",
    "    \n",
    "    kneedle = min(max(70, int(kneedle.knee * 6)), 350)\n",
    "    dict_knee_markers[clus] = dict_make_gene_scoring_cluster_robust_human[clus].iloc[:kneedle].index.values\n",
    "\n",
    "%store dict_knee_markers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Obtain summary table with information from 250 top markers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dict_info = make_combined_table(list_datasets=list_datasets_human, list_names=list_names_human, group_name='cluster_robust', list_clusters=list_accepted_clusters_human, dict_scoring_genes=dict_make_gene_scoring_cluster_robust_human, N=250)\n",
    "\n",
    "\n",
    "file_name = \"results/robust_markers_human.xlsx\"\n",
    "\n",
    "# Create an Excel writer object\n",
    "with pd.ExcelWriter(file_name, engine='xlsxwriter') as writer:\n",
    "    # Loop through the dictionary and write each DataFrame to a different sheet\n",
    "    df_base_explanation = pd.DataFrame(data=['Combined log10 p-value of all p-values from different datasets. The combined p-value is computed using scipy.stats.combine_pvalues with Stouffer method.', \n",
    "                   'Geometric mean of the log2 fold changes of individual datasets. The geometric mean was computed by applying 2^(list of FCs), and then it was log2-transformed.',\n",
    "                   'Mean values of scanpy scores computed through the sc.tl.rank_genes_groups function. The scores are averaged using a ponderation factor based on the expression (explained in Supplementary materials).', \n",
    "                   'Standard deviation of the scanpy scores powered as std^(1/3).', \n",
    "                   'Coefficient of variation, computed as mean/std.', \n",
    "                   'Averaged expression value (using the same weight method as with combined_score_mean), using the expression of the gene in each cell and in the k-nearest neighbours.',\n",
    "                   'Score calcuated as combined_score_mean / (gene_expr ^ (1/4)) (explained in Supplementary materials). This division by the gene expression favours the selection of genes the mid expression ranges.'],\n",
    "             index=['combined_log_pval', 'combined_LFC', 'combined_score_mean', 'combined_score_dev', 'CV', 'gene_expr', 'Z'])\n",
    "    df_base_explanation.to_excel(writer, sheet_name='reference', index=True, header=False)\n",
    "\n",
    "    for sheet_name, df in dict_info.items():\n",
    "        df.to_excel(writer, sheet_name=sheet_name.replace('/', '|'), index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analyse the presence and expression pattern of marker genes (within fibroblast populations)\n",
    "\n",
    "In this section we will plot UMAPs and dot plots of expression patterns of gene markers. This is useful because many gene markers tend to be markers of few clusters based on DE analysis but, in reality, they are also expressed in other clusters which might need to be considered."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "genes = ['A2M', 'AADAC', 'ABCA10', 'ABCA6', 'ABCA8', 'ABCA9', 'ABCC9', 'ABI3BP', 'ACAN', 'ACE', 'ACHE', 'ACKR2', 'ACKR3', 'ACKR4', 'ACTN1', 'ADA', 'ADAM12', 'ADAMTS1', 'ADAMTS18', 'ADAMTS2', 'ADAMTS4', 'ADAMTS5', 'ADAMTS9', 'ADAMTSL1', 'ADAMTSL3', 'ADAMTSL4', 'ADAMTSL5', 'ADCYAP1R1', 'ADGRD1', 'ADGRE2', 'ADGRE5', 'ADGRG2', 'ADRA2A', 'AEBP1', 'AFAP1L2', 'AGTR1', 'AHRR', 'AIF1L', 'AKAP12', 'AKAP6', 'ALDH1A3', 'ALPL', 'ALX4', 'ANGPTL1', 'ANGPTL5', 'ANGPTL7', 'ANKRD29', 'ANOS1', 'ANTXR1', 'ANTXR2', 'ANXA1', 'AOPEP', 'APBB1IP', 'APCDD1', 'APELA', 'APOC1', 'APOD', 'APOE', 'APP', 'AQP1', 'AQP3', 'ARF4', 'ARHGAP15', 'ARHGAP28', 'ARHGDIB', 'ARL4A', 'ARPC1B', 'ASPN', 'ATP10A', 'ATP1A2', 'ATP1B1', 'AXIN2', 'AXL', 'BAMBI', 'BASP1', 'BCL11B', 'BCL3', 'BGN', 'BHLHE40', 'BHLHE41', 'BIRC3', 'BMP1', 'BMP7', 'BMPER', 'BNC2', 'BST1', 'BTBD11', 'C10orf105', 'C11orf96', 'C19orf33', 'C1QTNF3', 'C1QTNF6', 'C1S', 'C1orf198', 'C2orf40/ECRG4', 'C3', 'C4orf48', 'C6', 'C7', 'C9orf3', 'CA12', 'CADM1', 'CADM2', 'CADM3', 'CALD1', 'CALM2', 'CAPN6', 'CAV1', 'CAV2', 'CAVIN2', 'CBFA2T3', 'CCBE1', 'CCDC146', 'CCDC80', 'CCDE80', 'CCK', 'CCL13', 'CCL19', 'CCL2', 'CCND1', 'CD109', 'CD151', 'CD200', 'CD200R1', 'CD248', 'CD34', 'CD40', 'CD47', 'CD55', 'CD63', 'CD70', 'CD74', 'CD81', 'CD9', 'CD99', 'CDA', 'CDH11', 'CDKN1A', 'CDKN2B', 'CENPW', 'CERCAM', 'CES1', 'CFD', 'CFH', 'CFHR1', 'CFL1', 'CGNL1', 'CGREF1', 'CH25H', 'CHADL', 'CHCHD10', 'CHD1', 'CHN1', 'CHPF', 'CHRDL1', 'CHST15', 'CILP', 'CLDN1', 'CLEC14A', 'CLEC2A', 'CLEC3B', 'CLSTN3', 'CNN2', 'CNN3', 'CNTN1', 'CNTN4', 'COCH', 'COL11A1', 'COL13A1', 'COL14A1', 'COL15A1', 'COL16A1', 'COL18A1', 'COL1A2', 'COL21A1', 'COL23A1', 'COL24A1', 'COL27A1', 'COL28A1', 'COL3A1', 'COL4A1', 'COL4A2', 'COL4A4', 'COL5A1', 'COL5A2', 'COL6A1', 'COL6A2', 'COL6A3', 'COL6A5', 'COL7A1', 'COL8A1', 'COL8A2', 'COL9A3', 'COMP', 'COPZ2', 'CORIN', 'COX4I2', 'CPE', 'CPNE5', 'CPVL', 'CPXM2', 'CPZ', 'CRABP1', 'CRABP2', 'CREB5', 'CRIP1', 'CRISPLD1', 'CRLF1', 'CSF1', 'CSPG4', 'CSRNP1', 'CSRP1', 'CSRP2', 'CTHRC1', 'CTNNAL1', 'CTSB', 'CTSH', 'CTSK', 'CTTNBP2', 'CX3CL1', 'CXCL1', 'CXCL12', 'CXCL2', 'CXCL3', 'CXCR4', 'CYBRD1', 'CYGB', 'CYP1B1', 'CYP26B1', 'CYP4B1', 'CYP7B1', 'CYYR1', 'DAAM2', 'DACT2', 'DBN1', 'DCN', 'DCXR', 'DDAH2', 'DDIT4', 'DEPTOR', 'DHRS3', 'DIO2', 'DKK2', 'DMD', 'DNAJA1', 'DOCK9', 'DOK6', 'DPEP1', 'DPP4', 'DPT', 'DUSP10', 'DUSP5', 'EBF2', 'ECM1', 'EDIL3', 'EDN3', 'EDNRA', 'EDNRB', 'EEF1A1', 'EFEMP1', 'EFNA1', 'EFNA5', 'EFNB1', 'EGFL6', 'EGFLAM', 'EGFR', 'EGR1', 'EGR2', 'EGR3', 'ELL2', 'ELN', 'EMB', 'EMID1', 'EMILIN2', 'EMP2', 'EMP3', 'EMX2', 'ENDOD1', 'ENHO', 'ENTPD2', 'EPHA3', 'EPHB6', 'EPS8', 'ERRFI1', 'ETV1', 'ETV4', 'EVA1A', 'EZR', 'F13A1', 'F2R', 'F2RL2', 'F3', 'F5', 'FABP3', 'FABP4', 'FABP5', 'FAM13A', 'FAM180B', 'FAM3C', 'FBLN1', 'FBLN2', 'FBN1', 'FBXO32', 'FGF13', 'FGF7', 'FGFBP2', 'FGFR2', 'FGL2', 'FHL2', 'FIBIN', 'FKBP9', 'FLNC', 'FMO1', 'FMO2', 'FMO3', 'FMOD', 'FNBP1L', 'FNDC1', 'FOS', 'FOSL1', 'FOXC2', 'FOXD2', 'FOXP1', 'FOXS1', 'FRMD4B', 'FRMD6', 'FST', 'FSTL1', 'FXYD6', 'FZD1', 'FZD2', 'FZD4', 'FZD6', 'FZD7', 'GAB1', 'GALNT15', 'GAS6', 'GATA3', 'GCH1', 'GDF10', 'GDF15', 'GEM', 'GFPT2', 'GFRA1', 'GFRA2', 'GGT5', 'GHR', 'GLRB', 'GLT8D2', 'GNAO1', 'GPC1', 'GPC3', 'GPM6B', 'GPNMB', 'GPX3', 'GREB1L', 'GREM2', 'GRN', 'GSN', 'HAPLN1', 'HAS1', 'HAS2', 'HBEGF', 'HEG1', 'HGF', 'HHIP', 'HIC1', 'HLA-B', 'HLA-C', 'HLA-DP1', 'HLA-DRB1', 'HLA-F', 'HMCN1', 'HMGA1', 'HMGCLL1', 'HMOX1', 'HOPX', 'HPGD', 'HPSE2', 'HRH1', 'HS3ST6', 'HSD3B7', 'HSPB3', 'HSPB6', 'HSPG2', 'HTRA1', 'HTRA3', 'ICAM1', 'ICAM2', 'ID4', 'IER3', 'IFI16', 'IGF1', 'IGFBP2', 'IGFBP3', 'IGFBP5', 'IGFBP6', 'IGFBP7', 'IGSF10', 'IL11RA', 'IL15', 'IL15RA', 'IL32', 'IL33', 'IL34', 'IL6', 'INHBA', 'INMT', 'IRF1', 'IRF8', 'ISLR', 'ISM1', 'ISYNA1', 'ITGA6', 'ITGB1', 'ITGB4', 'ITGBL1', 'ITIH5', 'ITM2A', 'ITM2B', 'JAG1', 'JAK3', 'JAM3', 'JUNB', 'KCNJ8', 'KCNK2', 'KCNQ3', 'KDELR3', 'KDM6B', 'KIAA1217', 'KIF26B', 'KLF5', 'KLK1', 'KLK4', 'KPNA2', 'KRT15', 'KRT17', 'KRT19', 'KRT31', 'KRT6A', 'KRT6B', 'KRT85', 'KTN1', 'LAMP1', 'LBH', 'LDB2', 'LDLR', 'LEF1', 'LEPR', 'LGMN', 'LGR4', 'LIMS2', 'LINC00327', 'LINC01133', 'LMO4', 'LMO7', 'LOX', 'LOXL1', 'LOXL2', 'LOXL4', 'LPAR4', 'LRIG1', 'LRP1', 'LRRC15', 'LRRC17', 'LRRN3', 'LRRN4CL', 'LSAMP', 'LSP1', 'LTBP2', 'LTBP4', 'LUM', 'LUZP2', 'LXN', 'MAB21L2', 'MAFB', 'MAFF', 'MAGI1', 'MAMDC2', 'MAP2', 'MAP7', 'MARCKSL1', 'MARVELD1', 'MATN2', 'MATN4', 'MCTP1', 'MDFIC', 'MDH', 'MDK', 'MEF2C', 'MEGF6', 'MEIS2', 'MEOX2', 'METRNL', 'MFAP2', 'MFAP4', 'MFAP5', 'MGLL', 'MGP', 'MGST1', 'MICAL2', 'MKX', 'MME', 'MMP11', 'MMP14', 'MMP16', 'MMP2', 'MMP27', 'MRAS', 'MRPS6', 'MSC', 'MTCL1', 'MTSS1', 'MTUS1', 'MXRA5', 'MXRA8', 'MYC', 'MYH9', 'MYO10', 'MYOC', 'NBL1', 'NCAM1', 'NDNF', 'NDP', 'NDRG1', 'NDRG2', 'NECAB1', 'NFATC1', 'NFATC2', 'NFE2L2', 'NFIB', 'NFIL3', 'NFKB1', 'NFKBIA', 'NFKBIZ', 'NGFR', 'NHSL1', 'NID1', 'NID2', 'NKD1', 'NKD2', 'NKI67', 'NLGN4X', 'NOCT', 'NOTUM', 'NOVA1', 'NPM1', 'NPTX2', 'NPY1R', 'NR1H3', 'NR2F1', 'NR2F2', 'NR4A3', 'NREP', 'NRN1', 'NRP1', 'NRP2', 'NTM', 'NUAK1', 'NUCB2', 'NXN', 'OGN', 'OLFML2A', 'OLFML2B', 'OMD', 'OSBP2', 'OSMR', 'P2RY14', 'P3H2', 'P4HA2', 'P4HA3', 'PAFAH1B3', 'PALLD', 'PALMD', 'PAM', 'PAMR1', 'PAPPA', 'PARD6G', 'PAWR', 'PCDH19', 'PCOLCE', 'PCOLCE2', 'PCSK1N', 'PCSK6', 'PCSK9', 'PDE1A', 'PDGFC', 'PDGFD', 'PDGFRA', 'PDGFRB', 'PDGFRL', 'PEAR1', 'PERP', 'PFN2', 'PGF', 'PGM2L1', 'PHACTR3', 'PHGDH', 'PHLDA1', 'PI16', 'PIEZO2', 'PIM1', 'PIM3', 'PKP4', 'PLA2G2A', 'PLA2G5', 'PLAT', 'PLAUR', 'PLCB1', 'PLEKHA4', 'PLEKHA6', 'PLK2', 'PLNXB2', 'PLPP1', 'PLPP5', 'PLSCR1', 'PLSCR4', 'PLTP', 'PLXDC1', 'PLXDC2', 'PLXNB2', 'PMEL', 'PMEPA1', 'PNP', 'PNRC1', 'PODN', 'PODNL1', 'POSTN', 'PPARG', 'PPDPFL', 'PPIB', 'PPIC', 'PPL', 'PPP1R14A', 'PPP1R15A', 'PPP1R15B', 'PRDM8', 'PRELP', 'PREX1', 'PREX2', 'PRG4', 'PRKG2', 'PRLR', 'PROCR', 'PROS1', 'PRR5L', 'PRSS23', 'PSAT1', 'PTCH1', 'PTCH2', 'PTGDS', 'PTGER3', 'PTGFR', 'PTGIS', 'PTGS1', 'PTGS1/COX1', 'PTGS2', 'PTH1R', 'PTK7', 'PTMA', 'PTN', 'PTPRD', 'PTPRE', 'PTPRK', 'PTX3', 'PXDNL', 'PXN', 'PYCR1', 'QPCT', 'RAB32', 'RAB34', 'RAMP1', 'RAMP2', 'RAMP3', 'RANBP3L', 'RARRES1', 'RARRES2', 'RBFOX1', 'RBP1', 'RBP4', 'RBP5', 'RCN3', 'REL', 'RGCC', 'RGS16', 'RHPN1', 'RNF152', 'ROBO1', 'ROBO2', 'ROR2', 'RRBP1', 'RSPO1', 'RSPO3', 'RSPO4', 'RUNX2', 'RUNX3', 'S100B', 'SATB2', 'SBSPON', 'SCARA3', 'SCARA5', 'SCARF2', 'SCN4B', 'SCN7A', 'SCRG1', 'SDC1', 'SDC2', 'SDC3', 'SDK1', 'SDPR/CAVIN2', 'SEMA3B', 'SEMA3C', 'SEMA3E', 'SERF2', 'SERPINA5', 'SERPINF1', 'SERPING1', 'SESN3', 'SFRP1', 'SFRP2', 'SFRP4', 'SGCA', 'SGIP1', 'SH3BGRL3', 'SHISA3', 'SHISAL1', 'SHOX2', 'SIRPA', 'SIX1', 'SLC16A3', 'SLC22A16', 'SLC22A3', 'SLC26A7', 'SLC29A1', 'SLC2A1', 'SLC2A3', 'SLC38A10', 'SLC40A1', 'SLC48A1', 'SLC5A3', 'SLC9A3R2', 'SLCO2B1', 'SLIT2', 'SLIT3', 'SLITRK6', 'SLPI', 'SMOC2', 'SMS', 'SMURF2', 'SNAI2', 'SNED1', 'SOCS3', 'SOD2', 'SORCS2', 'SOSTDC1', 'SOX4', 'SOX8', 'SOX9', 'SPARC', 'SPARCL1', 'SPHK1', 'SPON1', 'SPON2', 'SPRY1', 'SPRY2', 'SPSB1', 'SRPX', 'STC1', 'STC2', 'STMN1', 'STMN2', 'STXBP6', 'SULF2', 'SULT1A1', 'SUSD2', 'SVEP1', 'SYNE2', 'TAC1', 'TAGLN', 'TBX15', 'TBX3', 'TBXA2R', 'TCEAL2', 'TCF4', 'TCF7L2', 'TCHH', 'TENM2', 'TENM3', 'TFAP2A', 'TFAP2C', 'TGFB3', 'TGFBI', 'TGFBR2', 'TGFBR3', 'THBD', 'THBS2', 'THBS3', 'THBS4', 'THSD4', 'TIAM1', 'TIMP1', 'TIMP2', 'TIMP3', 'TJP1', 'TLN2', 'TM4SF1', 'TMEM119', 'TMEM135', 'TMEM150C', 'TMEM176A', 'TMEM176B', 'TMEM204', 'TMEM52', 'TMTC2', 'TNC', 'TNFAIP2', 'TNFAIP3', 'TNFAIP6', 'TNFRSF10B', 'TNFRSF11B', 'TNFRSF19', 'TNFSF10', 'TNFSF13B', 'TNFSF14', 'TNMD', 'TNN', 'TNNC1', 'TNNT3', 'TNS3', 'TNXB', 'TPD52', 'TPM2', 'TPPP3', 'TRAC', 'TRIB2', 'TRIL', 'TRIM47', 'TRIO', 'TRPM3', 'TRPS1', 'TSHZ3', 'TSPAN11', 'TSPAN13', 'TSPAN4', 'TSPAN7', 'TSPAN8', 'TTR', 'TUBA4A', 'TUBB4A', 'TWIST2', 'TXNIP', 'TYMP', 'UACA', 'UAP1', 'UBD', 'UCHL1', 'UGDH', 'UGT3A2', 'VASN', 'VCAM1', 'VCAN', 'VEGFA', 'VEGFB', 'VEGFD', 'VGLL3', 'VIPR2', 'VKORC1', 'VWA1', 'WFDC1', 'WIF1', 'WISP2', 'WISP2 / CCN5', 'WNT10A', 'WNT11', 'WNT2', 'WNT5A', 'WTAP', 'XG', 'YWHAH', 'ZC2HC1C', 'ZC3H12A', 'ZFP36', 'ZNF385A', 'ZNF608', 'ZYX']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To create the dot plot we will use two variables: the percentage of cells expressing the marker in each cluster, and the mean expression value of the expressing-cells in each cluster. To do that, we will create a df for each case and gene, compile the info for each adata and get the mean/std across the adatas. If one gene is not expressed in an adata, or the adata does not have a certain cluster, the value registered is a NaN, not zero. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot_UMAPS_gene('cluster_robust', list_datasets=list_all_datasets_human, list_names=list_names_human, n_cols=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "genes = ['ASPN']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dict_fraction_cells, dict_mean_exp = make_dicts_fraction_mean(genes, list_all_datasets=list_all_datasets_human, \n",
    "                                                              list_accepted_clusters=[i for i in list_accepted_clusters_human if i != \"T1\"], \n",
    "                                                              list_names=list_names_human, clusterby='cluster_robust')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To do the dotplot we do some adjustments. We adjust the fraction of cells beign expressed to $x^{2/3}$ to make it slightly more visual. We also apply a alpha of 1 - std(fraction of cells) to account for markers that are expressed in disparily, so that they are visually not that relevant.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for gene in genes:\n",
    "    print(gene)\n",
    "    plot_dotplot_gene(gene, dict_fraction_cells, dict_mean_exp)\n",
    "    plot_UMAPS_gene(gene, list_datasets=list_all_datasets_human, list_names=list_names_human, n_cols=5)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot_dotplot_list_genes(genes, dict_fraction_cells, dict_mean_exp, rotate=False, figsize=(10, len(genes) * 0.41))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analyse the presence and expression pattern of marker genes (within all populations)\n",
    "\n",
    "In this section we will plot UMAPs and dot plots of expression patterns of gene markers. This is useful because many gene markers tend to be markers of few clusters based on DE analysis but, in reality, they are also expressed in other clusters which might need to be considered."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "list_accepted_clusters_human_full = list(dict_cats_fb_human.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "list_all_datasets_human_full = [ahlers_2022_young_human, billi_2022_ctrl_human, boothby_2021_ctrl_human, burja_2022_ctrl_human,\n",
    "                         deng_2021_scar, gao_2021_ctrl_human, gaydosik_2020_ctrl_human, gur_2022_ctrl_human, \n",
    "                         he_2020_ctrl_human, hughes_2020_ctrl_human, kim_2020_ctrl_human, \n",
    "                         liu_2021_ctrl_human, mariottoni_2021_ctrl_human,\n",
    "                         mirizio_2020_scl_human, rindler_2021_ctrl_human,\n",
    "                         sole_2020_young_human, tabib_2018_ctrl_human, tabib_2021_ctrl_human, tsc_2021_ctrl_human,\n",
    "                         theo_2020_ctrl_human_dm, theo_2021_ctrl_human,\n",
    "                         vors_2020_ctrl_human, vors_2021_ctrl_human, xu_2021_ctrl_human]\n",
    "\n",
    "list_names_human_full = [adata.obs['Author'].values[0] + ' ' + str(int(adata.obs['Year'].values[0])) + ' human' for adata in list_all_datasets_human_full]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot_UMAPS_gene('assigned_cats', list_datasets=list_all_datasets_human_full, list_names=list_names_human_full, n_cols=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dict_fraction_cells_full, dict_mean_exp_full = make_dicts_fraction_mean(genes, list_all_datasets=list_all_datasets_human_full, list_names=list_names_human_full,\n",
    "                                                                        list_accepted_clusters=list_accepted_clusters_human_full, clusterby='assigned_cats')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for gene in genes:\n",
    "    print(gene)\n",
    "    plot_dotplot_gene(gene, dict_fraction_cells_full, dict_mean_exp_full, rotate=True)\n",
    "    plot_UMAPS_gene(gene, list_datasets=list_all_datasets_human_full, list_names=list_names_human, n_cols=5)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-09T11:59:21.139144Z",
     "iopub.status.busy": "2022-11-09T11:59:21.138621Z",
     "iopub.status.idle": "2022-11-09T11:59:21.171611Z",
     "shell.execute_reply": "2022-11-09T11:59:21.170714Z",
     "shell.execute_reply.started": "2022-11-09T11:59:21.139086Z"
    },
    "tags": []
   },
   "source": [
    "# Extracting gene markers based on manual notation from drive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "pd.set_option('display.max_rows', 500)\n",
    "from IPython.display import display, Markdown, Latex"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## A1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "cluster = 'A1'\n",
    "\n",
    "genes_A1_nice = ['AGTR1', 'ANGPTL5', 'ARFGEF3', 'CA12', 'CADM3', 'CD24', 'CD55', 'CD70', 'CDA', 'CGREF1', 'CLEC3B', 'CPVL', \n",
    "                 'CYBRD1', 'DKK1', 'DPP4', 'EYA2', 'FZD6', 'GALNT15', 'GDF15', 'GLRB', 'LGR5', 'LIMS2', 'MTCL1', 'NPR1', \n",
    "                 'RAB32', 'TUBB4A']\n",
    "genes_A1_fair = ['ABCB5', 'ABCC9', 'ACE', 'ACKR3', 'ACKR4', 'ADA', 'ADGRD1', 'AEBP1', 'AIF1L', 'AOX1', 'C1QTNF3', 'CD151', \n",
    "                 'CD248', 'CES1', 'CHRDL1', 'CILP', 'COL12A1', 'COL14A1', 'COL8A1', 'CORIN', 'CPZ', 'CTHRC1', 'CYP4B1', \n",
    "                 'DBN1', 'DCN', 'EMILIN2', 'ENTPD1', 'FABP3', 'FAM180B', 'FAP', 'FBLN1', 'FBLN2', 'FBN1', 'GPNMB', 'HPGD', \n",
    "                 'HSD3B7', 'ISLR', 'LEPR', 'LINC01133', 'LOX', 'LOXL1', 'MATN4', 'MEDAG', 'MFAP5', 'MGST1', 'MMP27', 'MTSS1', \n",
    "                 'NTN1', 'OMD', 'PAMR1', 'PCOLCE', 'PCOLCE2', 'PI16', 'PIEZO2', 'PODN', 'PRG4', 'PRKG2', 'QPCT', 'SCARA5', \n",
    "                 'SEMA3B', 'SEMA3C', 'SEMA3E', 'SERPINF1', 'SGCA', 'SGCG', 'SH3BGRL3', 'SLPI', 'SMOC2', 'THBS3', 'TNXB', \n",
    "                 'TPPP3', 'TSPAN8', 'WISP1/CCN4', 'WISP2 / CCN5', 'YWHAH']\n",
    "\n",
    "text_A1 = ''.join([f'***{i}*** ({df_human_genes_codes.loc[i, \"code\"].replace(cluster, f\"<u>{cluster}</u>\")}), ' \n",
    "                for i in genes_A1_nice] + ['\\n\\n'] +\\\n",
    "               [f'**{i}** ({df_human_genes_codes.loc[i, \"code\"].replace(cluster, f\"<u>{cluster}</u>\")}), ' \n",
    "                for i in genes_A1_fair])\n",
    "\n",
    "display(Markdown(text_A1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## A2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "cluster = 'A2'\n",
    "\n",
    "genes_A2_nice = ['AHRR', 'AKAP6', 'ANTXR1', 'APELA', 'C1orf198', 'CLEC2A', 'CMKLR1', 'COL13A1', 'COL18A1', 'COL23A1', \n",
    "                 'EDN3', 'EFNA5', 'F13A1', 'F5', 'FGFR2', 'HSPB3', 'KCNQ3', 'LINC00327', 'MAP2', 'MAP7', 'MOXD1', 'MYO1D', \n",
    "                 'NCKAP5', 'NTM', 'OSBP2', 'PCSK2', 'PHACTR3', 'PREX1', 'PTGS1/COX1', 'PTPRE', 'RAMP3', 'RGS2', 'RSPO3', \n",
    "                 'SCARF2', 'SHISAL1', 'STC1', 'STC2', 'THBD', 'TMEM52', 'TMTC2', 'TNFRSF19', 'TTR', ]\n",
    "genes_A2_fair = ['ACKR4', 'ADGRE2', 'APCDD1', 'AQP1', 'AXIN2', 'CCDC3', 'CCDC68', 'CD9', 'CDC42EP3', 'COL21A1', 'COL6A1', \n",
    "                 'COL6A2', 'COL6A3', 'COL6A5', 'COL7A1', 'COMP', 'CPXM2', 'CYP26B1', 'DAAM2', 'DKK3', 'DUSP6', 'EPHB6', \n",
    "                 'FNDC1', 'GREM2', 'HMGB3', 'IGFBP2', 'ISM1', 'KCNK2', 'LEPR', 'LOXL2', 'LSAMP', 'MAMDC2', 'NFIL3', \n",
    "                 'NKD1', 'NKD2', 'NPTX2', 'NTN1', 'PHLDA1', 'PTK7', 'RGS16', 'ROBO2', 'RSPO1', 'SFRP2' , 'SMIM3', \n",
    "                 'STMN1', 'TBX3', 'TCF4', 'THSD1', 'THSD4', 'TNC', 'TNFRSF21', 'WNT11']\n",
    "\n",
    "text_A2 = ''.join([f'***{i}*** ({df_human_genes_codes.loc[i, \"code\"].replace(cluster, f\"<u>{cluster}</u>\")}), ' \n",
    "                for i in genes_A2_nice] + ['\\n\\n'] +\\\n",
    "               [f'**{i}** ({df_human_genes_codes.loc[i, \"code\"].replace(cluster, f\"<u>{cluster}</u>\")}), ' \n",
    "                for i in genes_A2_fair])\n",
    "\n",
    "display(Markdown(text_A2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## A3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "cluster = 'A3'\n",
    "\n",
    "genes_A3_nice = ['ANGPTL5', 'ARFGEF3', 'CES1', 'COL14A1', 'CORIN', 'CPZ', 'ELN', 'ENPP1', 'SGCA', \n",
    "                 'SGCG', 'SLC47A2', 'SOSTDC1', 'WIF1',]\n",
    "genes_A3_fair = ['ADGRD1', 'AEBP1', 'ANGPTL1',  'AOX1', 'BGN', 'CADM3', 'CD9', 'CP', 'CPVL', 'ENTPD1', 'EPHB6', \n",
    "                 'FBLN1','FBLN2', 'GLRB', 'IGFBP4', 'ISM1', 'ITIH5', 'KCNK2', 'LOX', 'LOXL1','MMP27', 'NBL1', \n",
    "                 'NPTX2', 'OMD', 'PCOLCE', 'PDPN', 'QPCT', 'RSPO1','SERPINF1', 'SFRP2',  'SVEP1','THBS2', \n",
    "                 'THBS3', 'THBS4',  'XG']\n",
    "text_A3 = ''.join([f'***{i}*** ({df_human_genes_codes.loc[i, \"code\"].replace(cluster, f\"<u>{cluster}</u>\")}), ' \n",
    "                for i in genes_A3_nice] + ['\\n\\n'] +\\\n",
    "               [f'**{i}** ({df_human_genes_codes.loc[i, \"code\"].replace(cluster, f\"<u>{cluster}</u>\")}), ' \n",
    "                for i in genes_A3_fair])\n",
    "\n",
    "display(Markdown(text_A3))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## A4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "cluster = 'A4'\n",
    "\n",
    "genes_A4_nice = ['ACE', 'AIF1L', 'APCDD1L', 'C1QTNF3', 'CD248', 'CD34', 'CILP', 'DBN1', 'EMILIN2', 'EMP3', 'FABP3', \n",
    "                 'FAM180B', 'FBN1', 'HSD3B7', 'LINC01133', 'MATN4', 'PAMR1','PCOLCE2', 'PRG4', 'PTGIS', 'SEMA3C', \n",
    "                 'SEMA3E', 'SH3BGRL3', 'TPPP3', 'TRAC', 'WNT10B', 'YWHAH']\n",
    "genes_A4_fair = ['AADAC', 'ABCC9', 'ACKR3', 'AGTR1', 'AQP1', 'CA12', 'CD151', \n",
    "                 'CD24', 'CD55', 'CD70','CDA', 'CERCAM', 'CFD', 'CLEC3B', 'CLU', 'CRIP1', \n",
    "                 'CTHRC1','CYBRD1', 'DCN', 'DKK1', 'DPP4', 'EBF3', 'EFEMP1', \n",
    "                 'FAP', 'FNDC1', 'GALNT15', 'GPNMB','GPX3', 'GSN', 'IGFBP5', 'IGFBP6', \n",
    "                 'ISLR', 'LGR5', 'LIMS2', 'MEDAG', 'MFAP5', 'MMP2', 'NPR1', \n",
    "                 'PDGFRL', 'PI16', 'PIEZO2', 'PLA2G2A', 'PPIC', 'PRKG2', 'RAB32', 'SCARA5', 'SEMA3B', \n",
    "                 'SERPINA5', 'SLC29A1', 'SLPI', 'SMOC2', 'THY1', 'TIMP2','TNXB', 'TPM1', \n",
    "                 'TSPAN8', 'WISP2 / CCN5', 'WNT2', 'XG', ]\n",
    "\n",
    "text_A4 = ''.join([f'***{i}*** ({df_human_genes_codes.loc[i, \"code\"].replace(cluster, f\"<u>{cluster}</u>\")}), ' \n",
    "                for i in genes_A4_nice] + ['\\n\\n'] +\\\n",
    "               [f'**{i}** ({df_human_genes_codes.loc[i, \"code\"].replace(cluster, f\"<u>{cluster}</u>\")}), ' \n",
    "                for i in genes_A4_fair])\n",
    "\n",
    "display(Markdown(text_A4))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## B1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "cluster = 'B1'\n",
    "\n",
    "genes_B1_nice = ['ADAMTS4', 'APLNR', 'BAZ1A', 'C11orf96', 'CDKN1A', 'CHI3L1', 'CXCL1', 'CXCL2', \n",
    "                 'CXCL3', 'DYRK3', 'ELL2', 'ERRFI1', 'FOSL1', 'GCH1', 'GEM', 'HMGA1', 'IER3', 'IL6', 'KDM6B', 'KPNA2', \n",
    "                 'MMP1', 'MMP3', 'NFIL3', 'NFKB1', 'NR4A3', 'PPP1R15B', 'REL', 'TIPARP', 'TNFAIP6', 'TNFSF14', 'ZC3H12A', \n",
    "                 'ZNF331' ]\n",
    "genes_B1_fair = ['ACHE', 'ARID5B', 'AVPR1A', 'BIRC3', 'CCL2', 'CEBPB', 'CEBPD', 'CH25H', 'CHD1', 'COL4A4', 'DNAJA1', \n",
    "                 'EDNRB', 'EGR1', 'FGF7', 'FMO1', 'HLA-DRB1', 'HSPB6', 'ICAM1', 'IL32', 'IRF1', 'IRF8', 'JUNB', 'MAFF', \n",
    "                 'NFKBIA', 'PHLDA1', 'PIM3', 'PLAUR', 'PNRC1', 'PPDPFL', 'PPP1R15A', 'SMIM3', 'SNAI2', 'SOCS3', 'SOD2', \n",
    "                 'SPSB1', 'STC1', 'TNFAIP3', 'TNFRSF10B', 'TSHZ2', 'UAP1', 'WTAP', 'ZFP36']\n",
    "\n",
    "text_B1 = ''.join([f'***{i}*** ({df_human_genes_codes.loc[i, \"code\"].replace(cluster, f\"<u>{cluster}</u>\")}), ' \n",
    "                for i in genes_B1_nice] + ['\\n\\n'] +\\\n",
    "               [f'**{i}** ({df_human_genes_codes.loc[i, \"code\"].replace(cluster, f\"<u>{cluster}</u>\")}), ' \n",
    "                for i in genes_B1_fair])\n",
    "\n",
    "display(Markdown(text_B1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## B2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "cluster = 'B2'\n",
    "\n",
    "genes_B2_nice = ['ADRA2A', 'APOC1', 'APOE', 'CLSTN3', 'CXCL12', 'CYP7B1', 'HLA-B', 'ICAM2', 'IGFBP7', 'IL33', \n",
    "                 'IL34', 'JAK3', 'LPAR4', 'MSC', 'MUSK', 'PCDH17', 'PPDPFL', 'RBP5', 'SCN4B', 'SDK1', \n",
    "                 'SLC9A3R2', 'SLCO2B1', 'ST8SIA1', 'TMEM150C', 'TNFSF13B', 'TYMP', 'VCAM1']\n",
    "genes_B2_fair = ['ABCA10', 'ANKRD29', 'APLNR', 'C3', 'C6', 'C7', 'CCDC146', 'CCL19', 'CD200R1', 'CD74', \n",
    "                 'COL4A4', 'COX4I2', 'CRB2', 'CRB2', 'CTSH', 'CX3CL1', 'CYGB', 'EPHX1', 'FXYD6', 'GGT5', 'HLA-DRB1', \n",
    "                 'HLA-F', 'HMGCLL1', 'IGFBP3', 'LSAMP', 'NFIB', 'NLGN4X', 'NOTCH3', 'PTGDS', 'RARRES2', 'SLIT2', \n",
    "                 'THY1', 'TMEM176A', 'TMEM176B', 'TSHZ2', ]\n",
    "\n",
    "\n",
    "text_B2 = ''.join([f'***{i}*** ({df_human_genes_codes.loc[i, \"code\"].replace(cluster, f\"<u>{cluster}</u>\")}), ' \n",
    "                for i in genes_B2_nice] + ['\\n\\n'] +\\\n",
    "               [f'**{i}** ({df_human_genes_codes.loc[i, \"code\"].replace(cluster, f\"<u>{cluster}</u>\")}), ' \n",
    "                for i in genes_B2_fair])\n",
    "\n",
    "display(Markdown(text_B2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## B3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "cluster = 'B3'\n",
    "\n",
    "genes_B3_nice = ['ACHE', 'BIRC3', 'CCL19', 'CD74', 'CH25H', 'CSF1', 'CRB2', 'CTSH', 'CX3CL1', 'EGR1', 'EXOC3L4', 'HAS2', \n",
    "                 'HLA-B', 'HLA-F', 'IL15', 'IL4I1', 'IRF8', 'OLFM2', 'POPDC2', 'PTPRT', 'RGS16', 'TYMP', 'UBD', ]\n",
    "genes_B3_fair = ['ADAMTS4', 'ADRA2A', 'ANKRD29', 'APOE', 'AVPR1A', 'C3', 'CCDC146', 'CCL2', \n",
    "                 'CD200R1', 'CD82', 'CDKN1A', 'CLSTN3', 'COL6A5', 'COX4I2', \n",
    "                 'CXCL1', 'CXCL2', 'CXCL3', 'CYP7B1', 'FXYD6', 'GEM', \n",
    "                 'ICAM1', 'ICAM2', 'IL32', 'IL33', 'IL34', 'IRF1', 'JAK3', \n",
    "                 'JUNB', 'LPAR4', 'MSC', 'NFKBIA', 'NKD2', 'NLGN4X', 'NOTCH3', 'PCDH17', 'PIM3', \n",
    "                 'PPP1R15A', 'RBP5', 'SCN4B', 'SDK1', 'SLC2A3', 'SLC9A3R2', 'SLCO2B1', 'SLIT2', \n",
    "                 'SOCS3', 'SOD2', 'ST8SIA1', 'TMEM150C', 'TMEM176A', 'TMEM176B', 'TNC', 'TNFAIP3', 'TNFSF13B', \n",
    "                 'TNFSF13B', 'VCAM1', 'WTAP', 'ZC3H12A', 'ZFP36']\n",
    "\n",
    "\n",
    "text_B3 = ''.join([f'***{i}*** ({df_human_genes_codes.loc[i, \"code\"].replace(cluster, f\"<u>{cluster}</u>\")}), ' \n",
    "                for i in genes_B3_nice] + ['\\n\\n'] +\\\n",
    "               [f'**{i}** ({df_human_genes_codes.loc[i, \"code\"].replace(cluster, f\"<u>{cluster}</u>\")}), ' \n",
    "                for i in genes_B3_fair])\n",
    "\n",
    "display(Markdown(text_B3))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## B4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "cluster = 'B4'\n",
    "\n",
    "genes_B4_nice = ['AADAC', 'ADA',  'ATP8B4', 'C6', 'CD36', 'EFEMP1', 'FGF10', 'FGF7', 'FZD4', \n",
    "                 'GDF10', 'GPX3', 'HHIP', 'HSPB6', 'IGF2', 'MGP', 'MYOC', 'PHLDA3', 'PLA2G2A', \n",
    "                 'PLA2G5', 'PLBD1', 'PPARG', 'PPP1R1A', 'PQLC2L', 'SERPINA5', 'SIX1', 'SULT1A1', 'TNNT3', 'ZFHX4' ]\n",
    "genes_B4_fair = ['ABCA10', 'ABCA8', 'ABCA9', 'ADAMTSL3', 'APOC1', 'APOD', 'C11orf96', 'C7', 'CFH', 'CHRDL1', 'CXCL12', \n",
    "                 'CYGB', 'CYP4B1', 'EBF3', 'EPHX1', 'ERRFI1', 'FMO1', 'GGT5', 'GSN', 'HMGCLL1', 'IGF1', 'ITM2A', 'LMO3', \n",
    "                 'LSP1', 'MFAP4', 'MGST1', 'NFIB', 'NFKB1', 'NTRK2', 'PODN', 'PPP1R15B', 'RARRES2', 'SPSB1', 'STMN2', \n",
    "                 'TXNIP', 'WNT11']\n",
    "\n",
    "\n",
    "text_B4 = ''.join([f'***{i}*** ({df_human_genes_codes.loc[i, \"code\"].replace(cluster, f\"<u>{cluster}</u>\")}), ' \n",
    "                for i in genes_B4_nice] + ['\\n\\n'] +\\\n",
    "               [f'**{i}** ({df_human_genes_codes.loc[i, \"code\"].replace(cluster, f\"<u>{cluster}</u>\")}), ' \n",
    "                for i in genes_B4_fair])\n",
    "\n",
    "display(Markdown(text_B4))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## C1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "cluster = 'C1'\n",
    "\n",
    "genes_C1_nice = [ 'ACAN', 'ACTA2', 'ADAMTS18', 'ANOS1', 'ARHGAP44', 'BCL11B', 'CALD1', 'CFHR1', 'CNN2', 'COL11A1', \n",
    "                 'COL21A1', 'DOK6', 'DPEP1', 'EGFL6', 'EVA1A', 'F2RL2', 'GNAO1', 'HS6ST2', 'LGR4', 'LRRN3', 'MAGI1', \n",
    "                 'MEF2C', 'MME', 'PAFAH1B3', 'PTCH1', 'RANBP3L', 'RBFOX1', 'RBP4', 'SLC26A7', 'TMEM171', 'UGT3A2', \n",
    "                 'VIPR2', 'WDR86', 'WFDC1']\n",
    "genes_C1_fair = ['ADAM12', 'ADAMTS9', 'ALX4', 'APELA', 'ASPN', \n",
    "                 'C1QTNF6', 'CCDC3', 'CCND1', 'CDC42EP3', 'CDH11', 'COL12A1', \n",
    "                 'COL15A1', 'COL5A1', 'COL5A2', 'COL6A1', 'COL6A2', 'COL6A3', 'COL7A1', 'COL8A2', 'CPXM2', \n",
    "                 'CYYR1', 'DKK3', 'EDIL3', 'EDNRA', 'EDNRB', 'EGFLAM', 'EPHA3', 'F2R', \n",
    "                 'FBXO32', 'FZD2', 'GMFG', 'GPC1', 'GPC3', 'HAPLN1', 'KIAA1217', 'KIF26B', \n",
    "                 'KRT17', 'LAMC3', 'LEF1', 'LMO4', 'LRRC15', 'LRRC17', 'LTBP2', 'MDK', \n",
    "                 'MFAP2', 'MICAL2', 'MMP11', 'MXRA5', 'MYO10', 'NRP1', 'NTRK2', \n",
    "                 'PCDH19', 'PDE1A', 'PMEPA1', 'POSTN', 'PPP1R14A', 'PRSS23', 'RAMP1', 'RASL11B', \n",
    "                 'RFLNB', 'RGMA', 'RUNX2', 'SEMA5A', 'SGIP1', 'SOX18', 'SOX2', \n",
    "                 'SPARC', 'SPON1', 'STMN1', 'TBX15', 'TCF7L2', 'TENM3', 'THSD4', 'TNMD', 'TNN', 'TPM1', \n",
    "                 'TRIL', 'TSPAN13']\n",
    "\n",
    "text_C1 = ''.join([f'***{i}*** ({df_human_genes_codes.loc[i, \"code\"].replace(cluster, f\"<u>{cluster}</u>\")}), ' \n",
    "                for i in genes_C1_nice] + ['\\n\\n'] +\\\n",
    "               [f'**{i}** ({df_human_genes_codes.loc[i, \"code\"].replace(cluster, f\"<u>{cluster}</u>\")}), ' \n",
    "                for i in genes_C1_fair])\n",
    "\n",
    "display(Markdown(text_C1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## C2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "cluster = 'C2'\n",
    "\n",
    "genes_C2_nice = ['AOPEP', 'ARHGAP15', 'ATP1B1', 'BTBD11', 'CADM2', 'CCDC68', 'CCK', 'CHADL', 'CHST15', 'COCH', 'COL24A1', \n",
    "                 'CPNE5', 'CRABP1', 'EMB', 'FGF13', \n",
    "                 'FIBIN', 'GAP43', 'GAP43', 'GRP', 'HSPA2', 'KCNAB1', 'LRIG1', 'MAB21L2', 'MKX', 'NDNF', 'P3H2', 'PCSK9', \n",
    "                 'PLPP5', 'PLXDC1', 'PTPRD', 'PXDNL', 'RHPN1', 'SLC22A16', 'SLC40A1', 'SLITRK6', 'TCEAL2', 'TRIB2', 'TRPM3', \n",
    "                 'VEGFD', 'WNT10A', 'ZC2HC1C']\n",
    "genes_C2_fair = ['ADAMTS6', 'ANTXR2', 'APCDD1', 'ARID5B', 'ASPN', 'AXIN2', 'C9orf3', 'CADM1', \n",
    "                 'CD34', 'CLEC14A', 'CTSK', 'CYP1B1', 'CYYR1', 'DAAM2', 'DCXR', 'DKK2', 'EDNRA', 'EMID1', \n",
    "                 'F13A1', 'FMOD', 'FZD1', 'GPM6B', 'HPSE2', 'HTRA1', 'IGFBP5', 'KIF26B', 'LIMCH1', 'MAFA', \n",
    "                 'MAFB', 'MAMDC2', 'MEIS2', 'MEOX2', 'MOXD1', 'MXRA5', 'NCAM1', 'NDP', 'NECAB1', 'NFATC2', 'NOTUM', \n",
    "                 'NTM', 'OGN', 'PCSK1N', 'PDGFD', 'PGF', 'PODNL1', 'PRDM8', 'PTGFR', 'PTH1R', 'PTK7', 'PTPRK', 'RSPO4', \n",
    "                 'SDC1', 'SEMA3G', 'SFRP1', 'SPARCL1', 'TBX15', 'TCF4', 'THBS4', 'TNMD', 'TNN', 'TRPS1', \n",
    "                 'TWIST2']\n",
    "\n",
    "\n",
    "text_C2 = ''.join([f'***{i}*** ({df_human_genes_codes.loc[i, \"code\"].replace(cluster, f\"<u>{cluster}</u>\")}), ' \n",
    "                for i in genes_C2_nice] + ['\\n\\n'] +\\\n",
    "               [f'**{i}** ({df_human_genes_codes.loc[i, \"code\"].replace(cluster, f\"<u>{cluster}</u>\")}), ' \n",
    "                for i in genes_C2_fair])\n",
    "\n",
    "display(Markdown(text_C2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## C3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "cluster = 'C3'\n",
    "\n",
    "genes_C3_nice = ['ADAM12', 'BGN', 'C1QTNF6', 'EDIL3', 'HAPLN1', 'KLK4', 'MMP11', 'P4HA3', 'POSTN', 'RASL11B', 'RUNX2', \n",
    "                 'SEMA5A', 'SGIP1', 'SLC5A3', 'TAC1', 'TENM3', 'TSPAN13', 'WISP1/CCN4']\n",
    "genes_C3_fair = ['ACAN', 'ADGRE2', 'ASPN', 'CADM1', 'COL11A1', 'COL5A1', 'COL5A2', 'COL8A2', 'COMP', 'DKK2', 'DPT', \n",
    "                 'ELN', 'EMID1', 'F2R', 'FZD1', 'GPM6B', 'HTRA1', 'LMO7', 'LOXL2', 'LRRC15', 'LRRC17', 'LRRN3', 'LTBP2', \n",
    "                 'MAFB', 'MDK', 'MFAP2', 'MICAL2', 'MMP16', 'MRPS6', 'PMEPA1', 'PPP1R14A', 'PRSS23', 'PTN', 'RANBP3L', \n",
    "                 'RFLNB', 'RSPO4', 'SDC1', 'SPARC', 'TRIL']\n",
    "\n",
    "text_C3 = ''.join([f'***{i}*** ({df_human_genes_codes.loc[i, \"code\"].replace(cluster, f\"<u>{cluster}</u>\")}), ' \n",
    "                for i in genes_C3_nice] + ['\\n\\n'] +\\\n",
    "               [f'**{i}** ({df_human_genes_codes.loc[i, \"code\"].replace(cluster, f\"<u>{cluster}</u>\")}), ' \n",
    "                for i in genes_C3_fair])\n",
    "\n",
    "display(Markdown(text_C3))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## C5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "cluster = 'C5'\n",
    "\n",
    "genes_C5_nice = ['AOPEP', 'BMP7', 'CDH11', 'CENPW', 'CNTN4', 'CXCR4', 'FBXO32', 'GRIK1', 'IGFBP3', \n",
    "                 'IQGAP2', 'KRT9', 'LMO3', 'LUZP2', 'MAGI1', 'MARCKSL1', 'PDE3A', 'PGM2L1', 'PKP4', \n",
    "                 'PRDM1', 'PREX2', 'PRLR', 'SLC5A3', 'SOX18', 'TFAP2A', 'TFAP2C', 'TPD52', 'WNT5A']\n",
    "genes_C5_fair = ['ALX4', 'ARHGAP44', 'C9orf3', 'CPE', 'CRABP1', 'CYP26B1', 'DCXR', 'DIO2', 'EMP3', 'HEY2', 'HMGB3', \n",
    "                 'HPSE2', 'IER3', 'INHBA', 'JAG1', 'KRT17', 'LEF1', 'LMO4', 'MME', 'MRPS6', 'NCAM1', 'NDP', 'NKD1', \n",
    "                 'PAFAH1B3', 'PLAUR', 'PLK2', 'PTCH1', 'PTH1R', 'ROBO1', 'RSPO3', 'SEMA3G', 'SLC26A7', 'SNAI2', \n",
    "                 'SOX2', 'SPON2', 'TBX3', 'TRPS1']\n",
    "\n",
    "text_C5 = ''.join([f'***{i}*** ({df_human_genes_codes.loc[i, \"code\"].replace(cluster, f\"<u>{cluster}</u>\")}), ' \n",
    "                for i in genes_C5_nice] + ['\\n\\n'] +\\\n",
    "               [f'**{i}** ({df_human_genes_codes.loc[i, \"code\"].replace(cluster, f\"<u>{cluster}</u>\")}), ' \n",
    "                for i in genes_C5_fair])\n",
    "\n",
    "display(Markdown(text_C5))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## D1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "cluster = 'D1'\n",
    "\n",
    "genes_D1_nice = ['AFAP1L2', 'ALDH1A3', 'ANGPTL7', 'ATP1A2', 'BAMBI', 'C2orf40/ECRG4', 'CDH19', 'CHN1', \n",
    "                 'COL28A1', 'COL9A3', 'CRISPLD1', 'CYP1B1', 'DUSP5', 'EGR3', 'ENTPD2', 'ETV1', 'ETV4', \n",
    "                 'FMO2', 'FMO3', 'FOXC2', 'LZTS1', 'MCTP1', 'MRAS', 'MYBPC1', 'PODNL1', 'PSAT1', 'SCRG1', 'SOX8',\n",
    "                 'SOX9', 'TIAM1']\n",
    "\n",
    "genes_D1_fair = ['ABCA8', 'ABCA9', 'AKAP12', 'APOD', 'BMP7', 'BNC2', 'CAV1', 'CCL13', 'CD200', 'CFH', 'CHD1', 'CLDN1', \n",
    "                 'CNTN4', 'COL8A1', 'CSRP1', 'CTTNBP2', 'DAAM1', 'DDIT4L', 'DNAJA1', 'DUSP6', 'EBF2', 'EFNA1', \n",
    "                 'EFNB1', 'EGFR', 'FGFBP2', 'FLRT2', 'FOXS1', 'FZD2', 'GPC3', 'HMGA1', 'IGFBP7', 'INHBA', 'INMT', \n",
    "                 'ITGB4', 'KDM6B', 'KLK1', 'KPNA2', 'LDLR', 'LPCAT2', 'MAFF', 'MARCKSL1', 'MARCKSL1', 'MATN2', \n",
    "                 'MEOX2', 'MTSS1', 'MTUS1', 'NDRG2', 'NDUFA4L2', 'NR2F2', 'NRP2', 'OGN', 'OLFML2A', 'P2RY14', \n",
    "                 'PCDH19', 'PDZRN4', 'PEAR1', 'PLEKHA4', 'PLK2', 'PNRC1', 'PTGDS', 'REL', 'S100B', 'SBSPON', \n",
    "                 'SCN7A', 'SFRP4', 'SHISA3', 'SLC12A2', 'SLC2A1', 'SLC2A3', 'SPARCL1', 'SPRY2', 'STXBP6', 'TAGLN', \n",
    "                 'TENM2', 'TGFBI', 'TM4SF1', 'UACA', 'UAP1', 'VIT', 'WFDC1']\n",
    "\n",
    "\n",
    "text_D1 = ''.join([f'***{i}*** ({df_human_genes_codes.loc[i, \"code\"].replace(cluster, f\"<u>{cluster}</u>\")}), ' \n",
    "                for i in genes_D1_nice] + ['\\n\\n'] +\\\n",
    "               [f'**{i}** ({df_human_genes_codes.loc[i, \"code\"].replace(cluster, f\"<u>{cluster}</u>\")}), ' \n",
    "                for i in genes_D1_fair])\n",
    "\n",
    "display(Markdown(text_D1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## D2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "cluster = 'D2'\n",
    "\n",
    "genes_D2_nice = ['ADAMTSL3', 'ADAMTSL4', 'ADAMTSL5', 'AQP3', 'BNC2', 'C19orf33', 'CAV2', 'CAVIN2', 'CCL13', \n",
    "                 'CTTNBP2', 'DACT1', 'DOCK9', 'EGFLAM', 'GFRA2', 'GPC6', 'HRH1', 'IGF2', 'INMT', 'ISYNA1', 'ITGA6', \n",
    "                 'ITGB4', 'KLF5', 'KRT19', 'NDUFA4L2', 'NGFR', 'PALMD', 'PDZRN4', 'PEAR1', 'SBSPON', 'SDPR/CAVIN2', \n",
    "                 'SHISA3', 'SLC22A3', 'SLC2A1', 'TNNC1']\n",
    "\n",
    "genes_D2_fair = ['A2M', 'AKAP12', 'ALDH1A3', 'C2orf40/ECRG4', 'CAV1', 'CD200', 'CLDN1', 'COL9A3', 'CSRP1', 'DAAM1', \n",
    "                 'DDIT4L', 'DUSP5', 'EBF2', 'EFNA1', 'EFNB1', 'EGFR', 'FLRT2', 'FMO3', 'FOXS1', 'GAB1', 'GPC1', \n",
    "                 'IGFBP6', 'LMO7', 'MATN2', 'MRAS', 'MYBPC1', 'NDRG2', 'NR2F2', 'NRP2', 'P2RY14', 'PLEKHA4', 'S100B', \n",
    "                 'SCN7A', 'SFRP4', 'SLC12A2', 'SORBS1', 'STXBP6', 'TAGLN', 'TENM2', 'TGFBI', 'TJP1', 'TM4SF1', 'TXNIP', \n",
    "                 'UACA', 'VIT']\n",
    "\n",
    "text_D2 = ''.join([f'***{i}*** ({df_human_genes_codes.loc[i, \"code\"].replace(cluster, f\"<u>{cluster}</u>\")}), ' \n",
    "                for i in genes_D2_nice] + ['\\n\\n'] +\\\n",
    "               [f'**{i}** ({df_human_genes_codes.loc[i, \"code\"].replace(cluster, f\"<u>{cluster}</u>\")}), ' \n",
    "                for i in genes_D2_fair])\n",
    "\n",
    "display(Markdown(text_D2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## E1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "cluster = 'E1'\n",
    "\n",
    "genes_E1_nice = ['CMKLR1', 'COL26A1', 'EGR2', 'FHAD1', 'FZD7', 'GMFG', 'HOPX', 'IGFBP2', 'LFNG', 'MAP2', 'NECAB1', \n",
    "                 'NTRK3', 'PLA2G5', 'PLEKHA6', 'PRDM8', 'RAMP1', 'SAMD5', 'SIX1', 'SLC22A3', 'THSD1', ]\n",
    "genes_E1_fair = ['A2M', 'ANTXR2', 'COL15A1', 'CPE', 'DIO2', 'FGFBP2', 'IGF1', 'ITM2A', 'KIAA1217', 'KLK1', 'LAMC3', \n",
    "                 'LSP1', 'MEOX1', 'MGP', 'MMP16', 'OLFML2A', 'PDGFD', 'PGF', 'PTN', 'RGCC', 'RGMA', 'SFRP1', \n",
    "                 'SPON2', 'SPRY1', 'SPRY2', 'STMN2', 'TCF7L2', 'TIMP3', 'TMPO', 'TNFRSF21', 'UNC5B', 'WDR86', 'WNT2']\n",
    "\n",
    "text_E1 = ''.join([f'***{i}*** ({df_human_genes_codes.loc[i, \"code\"].replace(cluster, f\"<u>{cluster}</u>\")}), ' \n",
    "                for i in genes_E1_nice] + ['\\n\\n'] +\\\n",
    "               [f'**{i}** ({df_human_genes_codes.loc[i, \"code\"].replace(cluster, f\"<u>{cluster}</u>\")}), ' \n",
    "                for i in genes_E1_fair])\n",
    "\n",
    "display(Markdown(text_E1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dotplot with human markers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "markers_fb_human = [\n",
    "    'WISP2', 'SEMA3B', 'LGR5', 'ANGPTL5',\n",
    "    'COL18A1', 'NKD2', 'COL6A5', 'HSPB3', \n",
    "    'WIF1', 'SGCA', 'CORIN', 'SOSTDC1', \n",
    "    'C1QTNF3', 'SCARA5', 'PRG4', 'TRAC', \n",
    "    'GEM', 'CXCL2', 'TNFAIP6', 'CXCL1',\n",
    "    'GGT5', 'IL33', 'C7', 'SCN4B', \n",
    "    'CCL19', 'CTSH', 'RBP5', 'ACHE',\n",
    "    'EFEMP1', 'ITM2A', 'MYOC', 'GDF10', \n",
    "    'COL11A1', 'MEF2C', 'DPEP1', 'WFDC1',\n",
    "    'COCH', 'CRABP1', 'NDNF', 'SLITRK6',\n",
    "    'POSTN', 'BGN', 'LRRC15', 'LTBP2',\n",
    "    'IGFBP3', 'SLC5A3', 'WNT5A', 'LUZP2',\n",
    "    'ANGPTL7', 'ENTPD2', 'CDH19', 'ATP1A2',\n",
    "    'BNC2', 'ITGA6', 'ITGB4', 'TNNC1',\n",
    "    'IGFBP2', 'RAMP1', 'COL26A1', 'WNT2'\n",
    "    \n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dict_fraction_cells, dict_mean_exp = make_dicts_fraction_mean(markers_fb_human, list_all_datasets=list_all_datasets_human, \n",
    "                                                              list_accepted_clusters=[i for i in list_accepted_clusters_human if i != \"T1\"], \n",
    "                                                              list_names=list_names_human, clusterby='cluster_robust')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "ax = plot_dotplot_list_genes(markers_fb_human, dict_fraction_cells, \n",
    "                        dict_mean_exp,  rotate=False, do_return=True,\n",
    "                        figsize=(10, len(markers_fb_human) * 0.4))\n",
    "\n",
    "for idx, tick in enumerate(ax.get_yticklabels()):\n",
    "    if idx % 8 in [0, 1, 2, 3]:\n",
    "        tick.set_color(\"#171717\")\n",
    "    else:\n",
    "        tick.set_color(\"#777777\")\n",
    "        \n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "os.system('jupyter nbconvert --to html 4H_analysis_of_markers_and_pathways.ipynb')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "single-cell",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "384px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}