2H_comparison_human_populations.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Comparison of fibroblast populations\n",
    "\n",
    "In this notebook we are going to extract and replicate the main populations from diffrent papers where fibroblast populations are described, and find similarities and differences. The premise of this analysis is that many of the populations described in different papers seem not to match, or to be transcriptomically different, but in reality they are quite similar; that is, the main types of populations are indeed shared by the different papers, which should come as no surprise.\n",
    "\n",
    "Additionally, we will reanalize the *classic 4* papers, to check that cell populations are assigned as expected. For these papers, UMAPs might vary compared to the ones in our paper, but the main results should still be the same."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import scanpy as sc\n",
    "import scanpy.external as sce\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import triku as tk\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib as mpl\n",
    "from tqdm.notebook import tqdm\n",
    "import scipy.sparse as spr\n",
    "# import ray\n",
    "# import subprocess\n",
    "# import time\n",
    "# import scvelo as scv\n",
    "# import gc\n",
    "import gseapy as gp\n",
    "import pylab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!pip install cellassign"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from cellassign import assign_cats\n",
    "from fb_functions import clear_adata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# To print versions of imports \n",
    "\n",
    "import types\n",
    "\n",
    "def imports():\n",
    "    for name, val in globals().items():\n",
    "        if isinstance(val, types.ModuleType):\n",
    "            yield val.__name__\n",
    "\n",
    "excludes = ['builtins', 'types', 'sys']\n",
    "\n",
    "imported_modules = [module for module in imports() if module not in excludes]\n",
    "\n",
    "clean_modules = []\n",
    "\n",
    "for module in imported_modules:\n",
    "\n",
    "    sep = '.'  # to handle 'matplotlib.pyplot' cases\n",
    "    rest = module.split(sep, 1)[0]\n",
    "    clean_modules.append(rest)\n",
    "\n",
    "changed_imported_modules = list(set(clean_modules))  # drop duplicates\n",
    "\n",
    "pip_modules = !pip freeze  # you could also use `!conda list` with anaconda\n",
    "\n",
    "for module in pip_modules:\n",
    "    try:\n",
    "        name, version = module.split('==')\n",
    "        if name in changed_imported_modules:\n",
    "            print(name + '\\t' + version)\n",
    "    except:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "seed = 0\n",
    "%store seed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Palettes for UMAP gene expression\n",
    "\n",
    "magma = [plt.get_cmap('magma')(i) for i in np.linspace(0,1, 80)]\n",
    "magma[0] = (0.88, 0.88, 0.88, 1)\n",
    "magma = mpl.colors.LinearSegmentedColormap.from_list(\"\", magma[:65])\n",
    "\n",
    "%store magma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot_params = {\n",
    "            'figure.dpi': 300,\n",
    "            'legend.fontsize': 'large',\n",
    "            'axes.labelsize': 'x-large',\n",
    "            'axes.titlesize':'x-large',\n",
    "            'xtick.labelsize':'x-large',\n",
    "            'ytick.labelsize':'x-large', \n",
    "            'axes.grid': False, \n",
    "            'axes.facecolor':   'white',\n",
    "}\n",
    "\n",
    "%store plot_params\n",
    "\n",
    "pylab.rcParams.update(plot_params)\n",
    "pd.set_option('display.max_columns', None)\n",
    "pd.options.display.float_format = \"{:,.2f}\".format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dict_cats_fb_human = {'fibro': ['LUM', 'PDGFRA', 'COL1A1', 'DCN', 'SFRP2', 'APOE', 'APOD', 'FN1'], \n",
    "                'fibro - ANGPTL7': ['ANGPTL7', 'ENTPD2', 'ETV1', 'C2orf40', 'SCN7A', 'SOX8'],\n",
    "                'F': ['B4GALT1', 'TMSB4X', 'PPP1CB', 'WTAP', 'PTPRS', 'CTNNB1', 'INSR', 'BICC1', 'CTNNB1'], \n",
    "                'melanocyte': ['MLANA', 'PMEL', 'TRIM63', 'QPCT', 'PLP1', 'TYRP1'], \n",
    "                'neuro': ['GPM6B','PLP1','S100B','SCN7A','NRXN1','GFRA3','MPZ'],\n",
    "                'secretory cell': ['KRT7', 'KRT8', 'KRT18', 'KRT19', 'DCD', 'SCGB2A2', 'PPP1R1B', 'MUCL1', 'AZGP1', 'SCGB1D2', 'PDCD4', 'TSPAN8'], \n",
    "                # PERI / MUSCLE\n",
    "                'muscle': ['TAGLN', 'DES', 'PCP4', 'ACTG2', 'CNN1', 'CSRP1', 'TPM1', 'SYNPO2', 'PRUNE2', 'SORBS1', 'P2RX1'],\n",
    "                'peri - CYCS': ['TAGLN', 'CRISPLD2',  'CYCS', 'VDAC1', 'RHOB', 'SORBS2', 'PLEKHO1', 'CNN1', 'DNAJB9', 'CSRP2'],\n",
    "                'peri - RERGL': ['TAGLN', 'ACTA2', 'CRISPLD2',  'RERGL', 'BCAM', 'ADIRF', 'NET1', 'ARPC1A', 'PLN'],\n",
    "                'peri - RGS5': ['ACTA2',  'RGS5', 'ABCC9', 'HOPX', 'ARHGDIB', 'KCNJ8', 'FXYD6'],\n",
    "                'peri - ZFP36': [ 'RGS16', 'NR2F2', 'TGFBI', 'CCL8', 'RERG', 'HOPX'],\n",
    "                #---------------------------------\n",
    "                # ENDO INFO IS RECOLLECTED FROM LI et al 2021 - 10.7150/thno.54917\n",
    "                'endo artery': ['PLVAP', 'CLDN5', 'PECAM1', 'IGFBP3', 'SRGN', 'SEMA3G', 'RHOB', 'HEY1',], \n",
    "                'endo capillary': ['PLVAP', 'CLDN5', 'PECAM1', 'SELE', 'SOCS3', 'CDKN1A', 'NFKBIA', 'DNAJB1', 'ATF3',], \n",
    "                'endo venule': ['PLVAP', 'CLDN5', 'PECAM1', 'CYP1B1', 'CLU', 'PERP', 'VWF', 'IER3', 'TSC22D3'],                 \n",
    "                'lymph': ['CCL21', 'LYVE1', 'CLDN5', 'TFF3', 'MMRN1', 'EFEMP1', 'FGL2', 'TFPI', 'MAF'],\n",
    "                #----------------------------------------------------------\n",
    "                # KERATINOCYTE INFO - based on Cheng et al 2018 - 10.1016/j.celrep.2018.09.006\n",
    "                'krt basal': ['KRT14', 'COL17A1', 'KRT5', 'KRT15', 'DST', 'PDLIM1'], \n",
    "                'krt channel': ['KRT23', 'GJB6', 'GJB2', 'CDA', 'MMP7', 'PNLIPRP3'], \n",
    "                'krt spinous': ['KRT1', 'KRT10', 'DMKN', 'KRTDAP', 'CHP2', 'LYPD3'],\n",
    "                'krt gran': ['FLG', 'NCCRP1', 'CNFN', 'TGM1', 'CST6', 'KLK7'],\n",
    "                #----------------------------------------------\n",
    "                'immune': ['TPSB2', 'TPSAB1', 'HLA-DRA', 'FCER1G', 'CD74'], \n",
    "                'T CD4+': ['CD52', 'CD3D', 'TRAC', 'TCF7', 'CD4', 'IL7R', 'CD40LG'],\n",
    "                'T CD8+': ['CD52', 'CD3D', 'TRAC', 'CD8B', 'THEMIS', 'CD8A', 'FOXP3', 'CCR4', 'RORC', 'TIGIT',],\n",
    "                'B cells': ['IGHM','CD74','CD79A','NIBAN3','TCL1A','NCF1','MS4A1', 'BTK', 'CD19'],\n",
    "                'plasma cell': ['SDC1', 'SLAMF7', 'TNFRSF17', 'PTPRC', 'CXCR4', 'MYH9', 'PRDM1', 'CD38', 'CD27', 'IGHG1'],\n",
    "                'dendritic cell': ['GZMB', 'MRC1', 'XCR1', 'CLEC9A', 'IRF8', 'EPCAM', 'CD1B', 'STMN1', 'IDO1',],\n",
    "                'APC': ['HLA-DQA1', 'HLA-DRB6', 'TYROBP', 'FCER1G', 'AIF1'], \n",
    "                'mast cell': ['IL1RL1', 'CPA3', 'HPGDS', 'TPSB2', 'HPGD', 'RGS13', 'CTSG',  'TPSAB1', 'GATA2'],\n",
    "                'NK cell': ['NCAM1', 'XCL1', 'CD38', 'CD7', 'IL18R1', 'KLRF1', 'KLRK1'],\n",
    "                'mt': ['MTND2P28', 'MTND4P12', 'MTCO1P40', 'ADAM33', 'RN7SL2', 'MTRNR2L6'], \n",
    "                'eritro': ['HBB', 'HBA2', 'HBA1', 'HBD'],\n",
    "               }\n",
    "\n",
    "# We will use this in 4H\n",
    "%store dict_cats_fb_human  \n",
    "\n",
    "dict_cats_axes = {'A': ['PTGIS', 'SFRP2', 'MMP2', 'RGCC', 'COL14A1', 'AQP1', 'PTGIS', 'QPCT', 'ELN', 'COL14A1', 'LEPR', \n",
    "                        'ISM1', 'CES1', 'WIF1', 'RECK', 'SGCA', 'FBN1', 'NBL1', 'CTSB', 'COL5A1', 'SMOC2', 'SGCG'],\n",
    "                  'B': ['GGT5', 'APOE', 'APOC1', 'CYGB', 'C7', 'IGFBP7', 'TNFSF13B', 'APOC1', 'RARRES2', 'CCDC146', 'CXCL12', \n",
    "                        'ITM2A', 'EBF1', 'CCL2', 'IGFBP3', 'EBF1', 'CXCL2', 'EFEMP1', 'TMEM176A', 'C3', 'EGR1', 'LGALS3BP', \n",
    "                        'BST2', 'ANGPTL4', 'ABCA8', ],\n",
    "                  'C': ['TNN', 'TNMD', 'ASPN', 'CYP1B1', 'GPC1', 'PPP1R14A', 'GPM6B', 'COL11A1', 'DKK3', 'OGN', 'SDC1', \n",
    "                        'PDE1A', 'MDK', 'NRP2', 'POSTN', 'F2R', 'KIF26B', 'TENM3', 'ALX4', 'PMEPA1', 'FIBIN', 'PCDH15',]}\n",
    "\n",
    "# provisional_manual_dict_cats_clusters\n",
    "dict_cats_clusters= {\n",
    "                     'A1': ['SLPI', 'C1QTNF3', 'CD70', 'SEMA3C', 'TSPAN8', 'DPP4', 'CHRDL1', \n",
    "                             'SEMA3B', 'CTHRC1', 'WISP2', 'CCN5', 'PI16', 'QPCT'],\n",
    "                     'A2': ['CLEC2A', 'COL13A1', 'COL23A1', 'COL6A5', 'HSPB3', 'NPTX2', 'COMP', 'APCDD1', 'NKD2', \n",
    "                            'PHACTR3', 'AKAP6', 'LEPR', 'STC2', 'WIF1', 'CDC42EP3', 'COL18A1',], \n",
    "                     'A3': ['WIF1', 'HAS1', 'SGCG', 'CORIN', 'ACKR4', 'C1QTNF3', 'CFD', 'QPCT', 'SGCA', ], # 'HAS1', 'CORIN', 'SGCG', 'F13A1', 'RETREG1', \n",
    "                     'A4': ['SFRP4', 'HSD3B7', 'CD70', 'C1QTNF3', 'PRG4', 'GLIPR2', 'PAMR1', 'FNDC1',],\n",
    "                     'B1': ['MEDAG', 'CXCL1', 'IL6', 'ERRFI1', 'SPSB1', 'C11orf96', 'KDM6B', 'EIF4A3', 'NR4A2', 'ZC3H12A', \n",
    "                            'KDM6B', 'ZC3H12A', 'UAP1', 'CXCL2', 'SOD2', 'CSRNP1', 'ICAM1', 'MAFF', 'KPNA2', ],\n",
    "                     'B3': ['CCL19', 'CCL2', 'CDKN1A', 'CH25H', 'IL33', 'FOSB', 'EGR1', 'HES1', 'UBD', 'TNFAIP3', ],  # B0 = B1B2\n",
    "                     'B2': ['EGR1', 'CD74', 'IGFBP3', 'SPARCL1', 'IGFBP7', 'TMEM150C', 'P2RY14', 'IL34', 'PTGDS', 'APOC1', 'CYGB', \n",
    "                            'CTSH', 'TNFSF13B', 'CCDC146', 'C7', 'RBP5', 'CXCL12', 'PTGDS', 'ICAM2', 'RARRES2', 'LPAR4', 'COX4I2', ],\n",
    "                     'B4': ['ITM2A', 'GPC3', 'CHRDL1', 'GPX3', 'EFEMP1', 'MYOC', 'CXCL12', 'PLA2G2A', 'RARRES2',  'MGP', 'TSPAN8',\n",
    "                            'GSN', 'APOD', 'PLA2G5', 'CFH', 'GGT5', 'SIX1', 'ADH1B', 'GDF10', 'MGST1', \n",
    "                             ], \n",
    "                     'C1': ['DPEP1', 'UGT3A2', 'COL11A1', 'MME', 'RBP4', 'MYL4', 'WFDC1', 'CYYR1', 'EDNRA', \n",
    "                            'MEF2C', 'TNMD', 'CDH11', 'COL21A1', 'GPC3', 'KIF26B', 'PPP1R14A', 'EDIL3', 'SLC26A7' ],\n",
    "                     'C2': ['COCH', 'SLITRK6', 'MKX', 'CHADL', 'RSPO4', 'CRABP1', 'NDNF', 'SLC22A16', 'FIBIN', 'TNN', 'CCK', \n",
    "                            'RHPN1', 'MAB21L2', 'ASPN', 'OGN', 'PLXDC1', 'SPARCL1', 'PLPP5', 'WNT10A', 'NECAB1', 'TNMD' ], # 'SLITRK6', 'MKX'\n",
    "                     'C3': ['BGN', 'F2R', 'TNN', 'POSTN', 'GPM6B', 'PRSS23', 'FAP', 'EMID1', ],\n",
    "                     'C5': ['IGFBP3', 'LUZP2', 'CENPW', 'TFAP2A', 'TPD52', 'LEF1', 'CPE', 'BMP7', 'DIO3', 'ACTR3B', \n",
    "                            'BAMBI', 'INHBA', 'SERPINE2', 'WNT5A', 'BMP4', 'STMN1', 'SEMA3G', 'NOG', 'DCXR', 'EDNRA'],\n",
    "    \n",
    "                     'D1': ['ANGPTL7', 'SCN7A', 'C2orf40', 'APOD', 'CLDN1', 'CYP1B1', 'FGFBP2', 'MARCKSL1', 'PODNL1', 'KLK1', 'TM4SF1', \n",
    "                             'NRP2', 'COL8A1', 'P2RY14', 'SFRP4', 'MRAS', 'GPC3', 'ETV1', 'TIAM1', 'SPARCL1'], \n",
    "                     'D2': ['NGFR', 'CLDN1', 'SBSPON', 'TAGLN', 'TM4SF1', 'SLC2A1', 'SLC22A3', 'TNNC1', 'BNC2', 'KLF5', 'C2orf40', 'AQP3', \n",
    "                             'CSRP1', 'PALMD', 'SFRP4', 'PLEKHA4', 'NR2F2', 'ISYNA1', 'SCN7A', 'IGFBP6'],\n",
    "                     'E1': ['FGFBP2', 'CPE', 'OLFML2A', 'SLC22A3', 'IGFBP2', 'SPON2', 'APOD', 'EGR2', \n",
    "                            'RAMP1', 'IGF1', 'KLK1', 'RGMA', 'PDGFD', 'PRSS23', 'TIMP3',], \n",
    "    \n",
    "                     'T1': ['ASPN', 'MOB3B', 'PLEKHH2', 'PCDH15', 'RAI2', 'SPARCL1', 'TMEM176B', 'COL6A6', 'TMEM176A', \n",
    "                            'CYP1B1', 'MCTP2', 'BCL11A', 'LMO2', 'NTRK3', 'GLI2', 'PLPP4', 'CHN1', 'PDGFRL', 'IDE', 'LGR6', 'PI16', 'TELO2'],\n",
    "    \n",
    "                     'Glial': ['SOX10', 'S100B', 'NRNX1', 'L1CAM', 'AATK', 'SCN7A', 'GFRA3'], \n",
    "                     'Melanocyte': ['MLANA', 'PMEL', 'TRIM63', 'QPCT', 'PLP1', 'TYRP1'],\n",
    "                     'Endo': ['PLVAP', 'CLDN5', 'ACKR1', 'LMCD1', 'NPDC1', 'A2M', \n",
    "                         'PECAM1', 'CLU', 'VWF', 'CD74', 'RAMP2', 'IFI27', 'GNG11'], \n",
    "                      'Peri': ['RGS5', 'ITGA7', 'GJA4', 'MYH11', 'ANGPT2', 'LAMA5'], \n",
    "                    }  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dict_colors_human = {'A1': '#c93038', 'A2': '#efb241', 'A3': '#e38341', 'A4': '#740a2c',\n",
    "               'B1': '#d4f561', 'B2': '#2bb844', 'B3': '#7bdf46', 'B4': '#158858',\n",
    "               'C1': '#93dfe4', 'C2': '#63c2c9', 'C3': '#4c93ad', 'C5': '#264f6e',\n",
    "               'D1': '#ea387f', 'D2': '#fa7eaf', \n",
    "               'E1': '#b58057',\n",
    "               'T1': '#8e7cc3', 'U': '#dedede'}\n",
    "\n",
    "%store dict_colors_human"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dict_rep = {'CCN5': 'WISP2', 'ECRG4': 'C2orf40'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "mpl.rcParams['figure.dpi'] = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_score_graph(adatax):\n",
    "    df_cats_own = pd.DataFrame(index=adatax.obs_names, columns=['clusters', 'score'])\n",
    "    for cluster in adatax.obs['cluster'].cat.categories:\n",
    "        adata_sub = adatax[adatax.obs['cluster'] == cluster]\n",
    "        try:\n",
    "            df_cats_own.loc[adata_sub.obs_names, 'score'] = adata_sub.obs[f'cluster_{cluster}']\n",
    "            df_cats_own.loc[adata_sub.obs_names, 'clusters'] = cluster\n",
    "        except:\n",
    "            pass\n",
    "\n",
    "    df_cats_own = df_cats_own.sort_values('clusters')\n",
    "    sns.barplot(x='clusters', y='score', data=df_cats_own, palette=adatax.uns['cluster_colors'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "data_dir = os.getcwd() + '/data/'\n",
    "print(data_dir)\n",
    "%store data_dir"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create Ensemble ID dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "biomart_df = pd.read_csv('data/mart_export_human.txt', sep='\\t')\n",
    "biomart_df = biomart_df.dropna(how='any').drop_duplicates(subset='Gene name')\n",
    "dict_biomart_human = dict(zip(biomart_df['Gene name'], biomart_df['Gene stable ID']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**IMPORTANT: I am running this analysis in a computer with ~500 GB of RAM. I will load many datasets at once, which might be too much for some computers. I took this decision conciously, to have as much info available at any time as possible. If you cannot run all the analysis at once, you can run it by parts.**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## data extraction and processing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ahlers et al. 2022"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "ahlers_2022_dir = data_dir + '/ahlers_2022'\n",
    "os.makedirs(ahlers_2022_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "ahlers_2022_young_human = sc.read(ahlers_2022_dir + '/ahlers_2022_young_human.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "ahlers_2022_young_human.var_names = [dict_rep[i] if i in dict_rep else i for i in ahlers_2022_young_human.var_names]\n",
    "ahlers_2022_young_human.var['gene_symbol'] = ahlers_2022_young_human.var_names\n",
    "ahlers_2022_young_human.var['Ensemble ID'] = [dict_biomart_human[i] if i in dict_biomart_human else 'NA' for i in ahlers_2022_young_human.var_names]\n",
    "\n",
    "ahlers_2022_young_human.raw = ahlers_2022_young_human.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Basic QC filtering\n",
    "ahlers_2022_young_human.var['mt'] = ahlers_2022_young_human.var_names.str.startswith('MT-')  # annotate the group of mitochondrial genes as 'mt'\n",
    "sc.pp.calculate_qc_metrics(ahlers_2022_young_human, qc_vars=['mt'], percent_top=None, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.violin(ahlers_2022_young_human, ['n_genes_by_counts', 'total_counts', 'pct_counts_mt'],\n",
    "             jitter=0.4, multi_panel=True)\n",
    "\n",
    "sc.pl.scatter(ahlers_2022_young_human, x='total_counts', y='pct_counts_mt')\n",
    "sc.pl.scatter(ahlers_2022_young_human, x='total_counts', y='n_genes_by_counts')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1, 1, figsize=(10, 4))\n",
    "df = pd.DataFrame({'x': ahlers_2022_young_human.obs['Internal sample identifier'], 'y': ahlers_2022_young_human.obs['log1p_n_genes_by_counts']})\n",
    "sns.violinplot(x='x', y='y', data=df, ax=ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "ahlers_2022_young_human = ahlers_2022_young_human[(((ahlers_2022_young_human.obs['Internal sample identifier'] == 'P1_Y') & (ahlers_2022_young_human.obs.log1p_n_genes_by_counts < 7.5) & \n",
    "                                                (ahlers_2022_young_human.obs.log1p_n_genes_by_counts > 6.3)) | \n",
    "                                              ((ahlers_2022_young_human.obs['Internal sample identifier'] == 'P3_Y') & (ahlers_2022_young_human.obs.log1p_n_genes_by_counts < 7.5) & \n",
    "                                                (ahlers_2022_young_human.obs.log1p_n_genes_by_counts > 6.3)) | \n",
    "                                              ((ahlers_2022_young_human.obs['Internal sample identifier'] == 'P5_Y') & (ahlers_2022_young_human.obs.log1p_n_genes_by_counts < 7.5) & \n",
    "                                                (ahlers_2022_young_human.obs.log1p_n_genes_by_counts > 6.3)) \n",
    "                                             ).values, :]\n",
    "ahlers_2022_young_human = ahlers_2022_young_human[ahlers_2022_young_human.obs.pct_counts_mt < 7, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(ahlers_2022_young_human, min_counts=1)\n",
    "sc.pp.normalize_total(ahlers_2022_young_human)\n",
    "sc.pp.log1p(ahlers_2022_young_human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(ahlers_2022_young_human, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(ahlers_2022_young_human, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(ahlers_2022_young_human, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(ahlers_2022_young_human) ** 0.5 // 2), metric='cosine')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(ahlers_2022_young_human, min_dist=0.4, random_state=seed)\n",
    "sc.tl.leiden(ahlers_2022_young_human, resolution=8, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.subsample(ahlers_2022_young_human, fraction=1, random_state=0, copy=False)\n",
    "sc.pl.umap(ahlers_2022_young_human, color=['leiden', 'Internal sample identifier'], legend_loc='on data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(ahlers_2022_young_human, color=['leiden', 'LUM', 'PDGFRA', 'COL1A1', 'DCN', 'RGS5', 'MYH11', 'MLANA', 'PMEL', 'HBB'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(ahlers_2022_young_human, dict_cats=dict_cats_fb_human, min_score=0.45, quantile_gene_sel=0.97)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(ahlers_2022_young_human, color=['leiden', 'assigned_cats'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_fb_human.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(ahlers_2022_young_human, color=['assigned_cats'] + [i for i in val if i in ahlers_2022_young_human.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(ahlers_2022_young_human, color=['leiden', 'COL18A1', 'APCDD1', 'SLPI', 'CCL19', 'GGT5',  'CHRDL1', 'GPX3', 'BGN', 'ASPN', 'TNN', 'COL11A1', 'COCH', \n",
    "                                     'IGFBP3', 'ANGPTL7', 'SCN7A', 'C2orf40', 'NGFR', 'CLDN1', 'SBSPON', 'FGFBP2', 'DIO3', 'LUZP2'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "ahlers_2022_young_human_fb = ahlers_2022_young_human[ahlers_2022_young_human.obs['assigned_cats'].isin(['fibro', 'F', 'fibro - ANGPTL7'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(ahlers_2022_young_human_fb, min_counts=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(ahlers_2022_young_human_fb, random_state=seed, n_comps=50)\n",
    "sce.pp.harmony_integrate(ahlers_2022_young_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(ahlers_2022_young_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(len(ahlers_2022_young_human_fb) ** 0.5), metric='cosine')\n",
    "tk.tl.triku(ahlers_2022_young_human_fb, use_raw=False)\n",
    "sc.pp.pca(ahlers_2022_young_human_fb, random_state=seed, n_comps=50)\n",
    "sce.pp.harmony_integrate(ahlers_2022_young_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(ahlers_2022_young_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(len(ahlers_2022_young_human_fb) ** 0.5), metric='cosine')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(ahlers_2022_young_human_fb, min_dist=0.2, random_state=seed)\n",
    "sc.tl.leiden(ahlers_2022_young_human_fb, resolution=6, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(ahlers_2022_young_human_fb, dict_cats=dict_cats_clusters, min_score=0.5, quantile_gene_sel=0.95, key_added='cluster', others_name='U')\n",
    "assign_cats(ahlers_2022_young_human_fb, column_groupby='cluster', dict_cats=dict_cats_axes, min_score=0.5, quantile_gene_sel=0.9,  key_added='axis', \n",
    "             intermediate_states=True, diff=0.15, others_name='U')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "ahlers_2022_young_human_fb.uns['cluster_colors'] = [dict_colors_human[i] if i in dict_colors_human else '#bcbcbc' for \n",
    "                                      i in sorted(set(ahlers_2022_young_human_fb.obs['cluster']))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(ahlers_2022_young_human_fb, color=['Internal sample identifier', 'leiden', 'axis', 'cluster'], legend_loc='on data', cmap=magma, use_raw=False, ncols=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(ahlers_2022_young_human_fb, color=['Internal sample identifier', 'leiden', 'axis', 'cluster'], legend_loc='on data', cmap=magma, use_raw=False, ncols=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "del ahlers_2022_young_human_fb.obs['C3']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_clusters.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(ahlers_2022_young_human_fb, color=['cluster'] + [i for i in val if i in ahlers_2022_young_human_fb.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot_score_graph(ahlers_2022_young_human_fb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "clear_adata(ahlers_2022_young_human_fb)\n",
    "ahlers_2022_young_human_fb.write_h5ad(ahlers_2022_dir + '/ahlers_2022_young_human_fb_processed.h5')\n",
    "ahlers_2022_young_human.write_h5ad(ahlers_2022_dir + '/ahlers_2022_young_human_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "ahlers_2022_young_human = sc.read(ahlers_2022_dir + '/ahlers_2022_young_human_processed.h5')\n",
    "ahlers_2022_young_human_fb = sc.read(ahlers_2022_dir + '/ahlers_2022_young_human_fb_processed.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Alkon et al. 2022 [No fibroblasts in the sample?¿?]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "alkon_2022_dir = data_dir + '/alkon_2022'\n",
    "os.makedirs(alkon_2022_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "alkon_2022_ctrl_human = sc.read(alkon_2022_dir + '/alkon_2022_ctrl_human.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "alkon_2022_ctrl_human.var_names = [dict_rep[i] if i in dict_rep else i for i in alkon_2022_ctrl_human.var_names]\n",
    "alkon_2022_ctrl_human.var['gene_symbol'] = alkon_2022_ctrl_human.var_names\n",
    "alkon_2022_ctrl_human.var['Ensemble ID'] = [dict_biomart_human[i] if i in dict_biomart_human else 'NA' for i in alkon_2022_ctrl_human.var_names]\n",
    "\n",
    "alkon_2022_ctrl_human.raw = alkon_2022_ctrl_human.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Basic QC filtering\n",
    "alkon_2022_ctrl_human.var['mt'] = alkon_2022_ctrl_human.var_names.str.startswith('MT-')  # annotate the group of mitochondrial genes as 'mt'\n",
    "sc.pp.calculate_qc_metrics(alkon_2022_ctrl_human, qc_vars=['mt'], percent_top=None, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.violin(alkon_2022_ctrl_human, ['n_genes_by_counts', 'total_counts', 'pct_counts_mt'],\n",
    "             jitter=0.4, multi_panel=True)\n",
    "\n",
    "sc.pl.scatter(alkon_2022_ctrl_human, x='total_counts', y='pct_counts_mt')\n",
    "sc.pl.scatter(alkon_2022_ctrl_human, x='total_counts', y='n_genes_by_counts')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1, 1, figsize=(10, 4))\n",
    "df = pd.DataFrame({'x': alkon_2022_ctrl_human.obs['Internal sample identifier'], 'y': alkon_2022_ctrl_human.obs['log1p_n_genes_by_counts']})\n",
    "sns.violinplot(x='x', y='y', data=df, ax=ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "alkon_2022_ctrl_human = alkon_2022_ctrl_human[(((alkon_2022_ctrl_human.obs['Internal sample identifier'] == 'NHS1') & (alkon_2022_ctrl_human.obs.log1p_n_genes_by_counts < 6.8) & \n",
    "                                                (alkon_2022_ctrl_human.obs.log1p_n_genes_by_counts > 5.8)) | \n",
    "                                              ((alkon_2022_ctrl_human.obs['Internal sample identifier'] == 'NHS2') & (alkon_2022_ctrl_human.obs.log1p_n_genes_by_counts < 7) & \n",
    "                                                (alkon_2022_ctrl_human.obs.log1p_n_genes_by_counts > 6.3)) | \n",
    "                                              ((alkon_2022_ctrl_human.obs['Internal sample identifier'] == 'NHS3') & (alkon_2022_ctrl_human.obs.log1p_n_genes_by_counts < 8) & \n",
    "                                                (alkon_2022_ctrl_human.obs.log1p_n_genes_by_counts > 6)) \n",
    "                                             ).values, :]\n",
    "alkon_2022_ctrl_human = alkon_2022_ctrl_human[alkon_2022_ctrl_human.obs.pct_counts_mt < 10, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(alkon_2022_ctrl_human, min_counts=1)\n",
    "sc.pp.normalize_total(alkon_2022_ctrl_human)\n",
    "sc.pp.log1p(alkon_2022_ctrl_human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(alkon_2022_ctrl_human, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(alkon_2022_ctrl_human, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(alkon_2022_ctrl_human, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(alkon_2022_ctrl_human) ** 0.5 // 2), metric='cosine')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(alkon_2022_ctrl_human, min_dist=0.4, random_state=seed)\n",
    "sc.tl.leiden(alkon_2022_ctrl_human, resolution=8, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.subsample(alkon_2022_ctrl_human, fraction=1, random_state=0, copy=False)\n",
    "sc.pl.umap(alkon_2022_ctrl_human, color=['leiden', 'Internal sample identifier'], legend_loc='on data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(alkon_2022_ctrl_human, color=['leiden', 'LUM', 'PDGFRA', 'COL1A1', 'DCN', 'RGS5', 'MYH11', 'MLANA', 'PMEL', 'HBB'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(alkon_2022_ctrl_human, color=['leiden', 'LUM', 'PDGFRA', 'COL1A1', 'DCN', 'RGS5', 'MYH11', 'MLANA', 'PMEL', 'HBB'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(alkon_2022_ctrl_human, color=['leiden', 'CD79A', 'KRT1'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(alkon_2022_ctrl_human, dict_cats=dict_cats_fb_human, min_score=0.45, quantile_gene_sel=0.97)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(alkon_2022_ctrl_human, color=['leiden', 'assigned_cats'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Billi et al. 2022"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "billi_2022_dir = data_dir + '/billi_2022'\n",
    "os.makedirs(billi_2022_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "billi_2022_ctrl_human = sc.read(billi_2022_dir + '/billi_2022_ctrl_human.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "billi_2022_ctrl_human.var_names = [dict_rep[i] if i in dict_rep else i for i in billi_2022_ctrl_human.var_names]\n",
    "billi_2022_ctrl_human.var['gene_symbol'] = billi_2022_ctrl_human.var_names\n",
    "billi_2022_ctrl_human.var['Ensemble ID'] = [dict_biomart_human[i] if i in dict_biomart_human else 'NA' for i in billi_2022_ctrl_human.var_names]\n",
    "\n",
    "billi_2022_ctrl_human.raw = billi_2022_ctrl_human.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Basic QC filtering\n",
    "billi_2022_ctrl_human.var['mt'] = billi_2022_ctrl_human.var_names.str.startswith('MT-')  # annotate the group of mitochondrial genes as 'mt'\n",
    "sc.pp.calculate_qc_metrics(billi_2022_ctrl_human, qc_vars=['mt'], percent_top=None, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.violin(billi_2022_ctrl_human, ['n_genes_by_counts', 'total_counts', 'pct_counts_mt'],\n",
    "             jitter=0.4, multi_panel=True)\n",
    "\n",
    "sc.pl.scatter(billi_2022_ctrl_human, x='total_counts', y='pct_counts_mt')\n",
    "sc.pl.scatter(billi_2022_ctrl_human, x='total_counts', y='n_genes_by_counts')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1, 1, figsize=(10, 4))\n",
    "df = pd.DataFrame({'x': billi_2022_ctrl_human.obs['Internal sample identifier'], 'y': billi_2022_ctrl_human.obs['log1p_n_genes_by_counts']})\n",
    "sns.violinplot(x='x', y='y', data=df, ax=ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "billi_2022_ctrl_human = billi_2022_ctrl_human[((billi_2022_ctrl_human.obs.log1p_n_genes_by_counts < 8.5) & \n",
    "                                                (billi_2022_ctrl_human.obs.log1p_n_genes_by_counts > 7)).values, :]\n",
    "billi_2022_ctrl_human = billi_2022_ctrl_human[billi_2022_ctrl_human.obs.pct_counts_mt < 10, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(billi_2022_ctrl_human, min_counts=1)\n",
    "sc.pp.normalize_total(billi_2022_ctrl_human)\n",
    "sc.pp.log1p(billi_2022_ctrl_human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(billi_2022_ctrl_human, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(billi_2022_ctrl_human, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(billi_2022_ctrl_human, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(billi_2022_ctrl_human) ** 0.5 // 2), metric='cosine')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(billi_2022_ctrl_human, min_dist=0.4, random_state=seed)\n",
    "sc.tl.leiden(billi_2022_ctrl_human, resolution=8, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.subsample(billi_2022_ctrl_human, fraction=1, random_state=0, copy=False)\n",
    "sc.pl.umap(billi_2022_ctrl_human, color=['leiden', 'Internal sample identifier'], legend_loc='on data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(billi_2022_ctrl_human, color=['leiden', 'LUM', 'PDGFRA', 'COL1A1', 'DCN', 'RGS5', 'MYH11', 'MLANA', 'PMEL', 'HBB'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(billi_2022_ctrl_human, dict_cats=dict_cats_fb_human, min_score=0.45, quantile_gene_sel=0.97)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(billi_2022_ctrl_human, color=['leiden', 'assigned_cats'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(billi_2022_ctrl_human, color=['leiden', 'COL18A1', 'APCDD1', 'SLPI', 'CCL19', 'GGT5',  'CHRDL1', 'GPX3', 'BGN', 'ASPN', 'TNN', 'COL11A1', 'COCH', \n",
    "                                     'IGFBP3', 'ANGPTL7', 'SCN7A', 'C2orf40', 'NGFR', 'CLDN1', 'SBSPON', 'FGFBP2', 'DIO3', 'LUZP2'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "billi_2022_ctrl_human_fb = billi_2022_ctrl_human[billi_2022_ctrl_human.obs['assigned_cats'].isin(['fibro', 'F', 'fibro - ANGPTL7'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(billi_2022_ctrl_human_fb, min_counts=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(billi_2022_ctrl_human_fb, random_state=seed, n_comps=50)\n",
    "sce.pp.harmony_integrate(billi_2022_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(billi_2022_ctrl_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(0.3 * len(billi_2022_ctrl_human_fb) ** 0.5), metric='cosine')\n",
    "tk.tl.triku(billi_2022_ctrl_human_fb, use_raw=False)\n",
    "sc.pp.pca(billi_2022_ctrl_human_fb, random_state=seed, n_comps=50)\n",
    "sce.pp.harmony_integrate(billi_2022_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(billi_2022_ctrl_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(0.3 * len(billi_2022_ctrl_human_fb) ** 0.5), metric='cosine')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(billi_2022_ctrl_human_fb, min_dist=0.1, random_state=seed)\n",
    "sc.tl.leiden(billi_2022_ctrl_human_fb, resolution=12, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(billi_2022_ctrl_human_fb, dict_cats=dict_cats_clusters, min_score=0.5, quantile_gene_sel=0.65, key_added='cluster', others_name='U')\n",
    "assign_cats(billi_2022_ctrl_human_fb, column_groupby='cluster', dict_cats=dict_cats_axes, min_score=0.5, quantile_gene_sel=0.9,  key_added='axis', \n",
    "             intermediate_states=True, diff=0.15, others_name='U')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "billi_2022_ctrl_human_fb.uns['cluster_colors'] = [dict_colors_human[i] if i in dict_colors_human else '#bcbcbc' for \n",
    "                                      i in sorted(set(billi_2022_ctrl_human_fb.obs['cluster']))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(billi_2022_ctrl_human_fb, color=['Internal sample identifier', 'leiden', 'axis', 'cluster'], legend_loc='on data', cmap=magma, use_raw=False, ncols=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "del billi_2022_ctrl_human_fb.obs['C3']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_clusters.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(billi_2022_ctrl_human_fb, color=['cluster'] + [i for i in val if i in billi_2022_ctrl_human_fb.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot_score_graph(billi_2022_ctrl_human_fb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "clear_adata(billi_2022_ctrl_human_fb)\n",
    "billi_2022_ctrl_human_fb.write_h5ad(billi_2022_dir + '/billi_2022_ctrl_human_fb_processed.h5')\n",
    "billi_2022_ctrl_human.write_h5ad(billi_2022_dir + '/billi_2022_ctrl_human_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "billi_2022_ctrl_human = sc.read(billi_2022_dir + '/billi_2022_ctrl_human_processed.h5')\n",
    "billi_2022_ctrl_human_fb = sc.read(billi_2022_dir + '/billi_2022_ctrl_human_fb_processed.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Boothby et al. 2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "boothby_2021_dir = data_dir + '/boothby_2021'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "boothby_2021_ctrl_human = sc.read(boothby_2021_dir + '/boothby_2021_ctrl_human.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "boothby_2021_ctrl_human.var_names = [dict_rep[i] if i in dict_rep else i for i in boothby_2021_ctrl_human.var_names]\n",
    "boothby_2021_ctrl_human.var['gene_symbol'] = boothby_2021_ctrl_human.var_names\n",
    "boothby_2021_ctrl_human.var['Ensemble ID'] = [dict_biomart_human[i] if i in dict_biomart_human else 'NA' for i in boothby_2021_ctrl_human.var_names]\n",
    "\n",
    "boothby_2021_ctrl_human.raw = boothby_2021_ctrl_human.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Basic QC filtering\n",
    "boothby_2021_ctrl_human.var['mt'] = boothby_2021_ctrl_human.var_names.str.startswith('MT-')  # annotate the group of mitochondrial genes as 'mt'\n",
    "sc.pp.calculate_qc_metrics(boothby_2021_ctrl_human, qc_vars=['mt'], percent_top=None, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.violin(boothby_2021_ctrl_human, ['n_genes_by_counts', 'total_counts', 'pct_counts_mt'],\n",
    "             jitter=0.4, multi_panel=True)\n",
    "\n",
    "sc.pl.scatter(boothby_2021_ctrl_human, x='total_counts', y='pct_counts_mt')\n",
    "sc.pl.scatter(boothby_2021_ctrl_human, x='total_counts', y='n_genes_by_counts')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1, 1, figsize=(10, 4))\n",
    "df = pd.DataFrame({'x': boothby_2021_ctrl_human.obs['Internal sample identifier'], 'y': boothby_2021_ctrl_human.obs['log1p_n_genes_by_counts']})\n",
    "sns.violinplot(x='x', y='y', data=df, ax=ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "boothby_2021_ctrl_human = boothby_2021_ctrl_human[(((boothby_2021_ctrl_human.obs['Internal sample identifier'] == 'HC01') & (boothby_2021_ctrl_human.obs.log1p_n_genes_by_counts < 8) & \n",
    "                                                (boothby_2021_ctrl_human.obs.log1p_n_genes_by_counts > 7.3)) | \n",
    "                                              ((boothby_2021_ctrl_human.obs['Internal sample identifier'] == 'HC02') & (boothby_2021_ctrl_human.obs.log1p_n_genes_by_counts < 8) & \n",
    "                                                (boothby_2021_ctrl_human.obs.log1p_n_genes_by_counts > 7.2)) | \n",
    "                                              ((boothby_2021_ctrl_human.obs['Internal sample identifier'] == 'HC03') & (boothby_2021_ctrl_human.obs.log1p_n_genes_by_counts < 8) & \n",
    "                                                (boothby_2021_ctrl_human.obs.log1p_n_genes_by_counts > 7.3)) \n",
    "                                             ).values, :]\n",
    "boothby_2021_ctrl_human = boothby_2021_ctrl_human[boothby_2021_ctrl_human.obs.pct_counts_mt < 7, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(boothby_2021_ctrl_human, min_counts=1)\n",
    "sc.pp.normalize_total(boothby_2021_ctrl_human)\n",
    "sc.pp.log1p(boothby_2021_ctrl_human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(boothby_2021_ctrl_human, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(boothby_2021_ctrl_human, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(boothby_2021_ctrl_human, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(boothby_2021_ctrl_human) ** 0.5 // 2), metric='cosine')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(boothby_2021_ctrl_human, min_dist=0.4, random_state=seed)\n",
    "sc.tl.leiden(boothby_2021_ctrl_human, resolution=5, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.subsample(boothby_2021_ctrl_human, fraction=1, random_state=0, copy=False)\n",
    "sc.pl.umap(boothby_2021_ctrl_human, color=['leiden', 'Internal sample identifier'], legend_loc='on data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(boothby_2021_ctrl_human, color=['leiden', 'LUM', 'PDGFRA', 'COL1A1', 'DCN', 'RGS5', 'MYH11', 'MLANA', 'PMEL', 'HBB', 'S100B'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(boothby_2021_ctrl_human, dict_cats=dict_cats_fb_human, min_score=0.4, quantile_gene_sel=0.6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(boothby_2021_ctrl_human, color=['leiden', 'assigned_cats'] + dict_cats_fb_human['peri - CYCS'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_fb_human.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(boothby_2021_ctrl_human, color=['assigned_cats'] + [i for i in val if i in boothby_2021_ctrl_human.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(boothby_2021_ctrl_human, color=['leiden', 'COL18A1', 'APCDD1', 'SLPI', 'CCL19', 'GGT5',  'CHRDL1', 'GPX3', 'BGN', 'ASPN', 'TNN', 'COL11A1', 'COCH', \n",
    "                                     'IGFBP3', 'ANGPTL7', 'SCN7A', 'C2orf40', 'NGFR', 'CLDN1', 'SBSPON', 'FGFBP2', 'DIO3', 'LUZP2'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "boothby_2021_ctrl_human_fb = boothby_2021_ctrl_human[boothby_2021_ctrl_human.obs['assigned_cats'].isin(['fibro', 'F'])].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(boothby_2021_ctrl_human_fb, min_counts=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(boothby_2021_ctrl_human_fb, random_state=seed, n_comps=50)\n",
    "sce.pp.harmony_integrate(boothby_2021_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(boothby_2021_ctrl_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(0.1 * len(boothby_2021_ctrl_human_fb) ** 0.5), metric='cosine')\n",
    "tk.tl.triku(boothby_2021_ctrl_human_fb, use_raw=False)\n",
    "\n",
    "sc.pp.pca(boothby_2021_ctrl_human_fb, random_state=seed, n_comps=50)\n",
    "sce.pp.harmony_integrate(boothby_2021_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(boothby_2021_ctrl_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(0.1 * len(boothby_2021_ctrl_human_fb) ** 0.5), metric='cosine')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(boothby_2021_ctrl_human_fb, min_dist=0.25, random_state=seed)\n",
    "sc.tl.leiden(boothby_2021_ctrl_human_fb, resolution=9, random_state=seed)  # leiden 9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(boothby_2021_ctrl_human_fb, dict_cats=dict_cats_clusters, min_score=0.6, quantile_gene_sel=0.7, key_added='cluster', others_name='U')\n",
    "assign_cats(boothby_2021_ctrl_human_fb, column_groupby='cluster', dict_cats=dict_cats_axes, min_score=0.5, quantile_gene_sel=0.9,  key_added='axis', \n",
    "             intermediate_states=True, diff=0.15, others_name='U')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "boothby_2021_ctrl_human_fb.uns['cluster_colors'] = [dict_colors_human[i] if i in dict_colors_human else '#bcbcbc' for \n",
    "                                      i in sorted(set(boothby_2021_ctrl_human_fb.obs['cluster']))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(boothby_2021_ctrl_human_fb, color=['Internal sample identifier', 'leiden', 'axis', 'cluster'], legend_loc='on data', cmap=magma, use_raw=False, ncols=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "del boothby_2021_ctrl_human_fb.obs['C3']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_clusters.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(boothby_2021_ctrl_human_fb, color=['cluster'] + [i for i in val if i in boothby_2021_ctrl_human_fb.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot_score_graph(boothby_2021_ctrl_human_fb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "clear_adata(boothby_2021_ctrl_human_fb)\n",
    "boothby_2021_ctrl_human_fb.write_h5ad(boothby_2021_dir + '/boothby_2021_ctrl_human_fb_processed.h5')\n",
    "boothby_2021_ctrl_human.write_h5ad(boothby_2021_dir + '/boothby_2021_ctrl_human_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "boothby_2021_ctrl_human_fb = sc.read(boothby_2021_dir + '/boothby_2021_ctrl_human_fb_processed.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**U** cluster *BNIP3*, *BNIP3L*, *HILPDA*, *ERO1A*, *ENO2* indicates a possible hypoxia state in these cells."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.rank_genes_groups(boothby_2021_ctrl_human_fb, groupby='leiden', groups=['5'], reference='rest')\n",
    "sc.pl.rank_genes_groups_tracksplot(boothby_2021_ctrl_human_fb, dendrogram=False, n_genes=200)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Burja et al. 2022"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "burja_2022_dir = data_dir + '/burja_2022'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "burja_2022_ctrl_human = sc.read(burja_2022_dir + '/burja_2022_ctrl_human.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "burja_2022_ctrl_human.var_names = [dict_rep[i] if i in dict_rep else i for i in burja_2022_ctrl_human.var_names]\n",
    "burja_2022_ctrl_human.var['gene_symbol'] = burja_2022_ctrl_human.var_names\n",
    "burja_2022_ctrl_human.var['Ensemble ID'] = [dict_biomart_human[i] if i in dict_biomart_human else 'NA' for i in burja_2022_ctrl_human.var_names]\n",
    "\n",
    "burja_2022_ctrl_human.raw = burja_2022_ctrl_human.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Basic QC filtering\n",
    "burja_2022_ctrl_human.var['mt'] = burja_2022_ctrl_human.var_names.str.startswith('MT-')  # annotate the group of mitochondrial genes as 'mt'\n",
    "sc.pp.calculate_qc_metrics(burja_2022_ctrl_human, qc_vars=['mt'], percent_top=None, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.violin(burja_2022_ctrl_human, ['n_genes_by_counts', 'total_counts', 'pct_counts_mt'],\n",
    "             jitter=0.4, multi_panel=True)\n",
    "\n",
    "sc.pl.scatter(burja_2022_ctrl_human, x='total_counts', y='pct_counts_mt')\n",
    "sc.pl.scatter(burja_2022_ctrl_human, x='total_counts', y='n_genes_by_counts')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1, 1, figsize=(10, 4))\n",
    "df = pd.DataFrame({'x': burja_2022_ctrl_human.obs['Internal sample identifier'], 'y': burja_2022_ctrl_human.obs['log1p_n_genes_by_counts']})\n",
    "sns.violinplot(x='x', y='y', data=df, ax=ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "burja_2022_ctrl_human = burja_2022_ctrl_human[(((burja_2022_ctrl_human.obs['Internal sample identifier'] == 'HC 1') & (burja_2022_ctrl_human.obs.log1p_n_genes_by_counts < 8.5) & \n",
    "                                                (burja_2022_ctrl_human.obs.log1p_n_genes_by_counts > 7.4)) | \n",
    "                                              ((burja_2022_ctrl_human.obs['Internal sample identifier'] == 'HC 2') & (burja_2022_ctrl_human.obs.log1p_n_genes_by_counts < 8.5) & \n",
    "                                                (burja_2022_ctrl_human.obs.log1p_n_genes_by_counts > 7.4))\n",
    "                                             ).values, :]\n",
    "burja_2022_ctrl_human = burja_2022_ctrl_human[burja_2022_ctrl_human.obs.pct_counts_mt < 40, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(burja_2022_ctrl_human, min_counts=1)\n",
    "sc.pp.normalize_total(burja_2022_ctrl_human)\n",
    "sc.pp.log1p(burja_2022_ctrl_human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(burja_2022_ctrl_human, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(burja_2022_ctrl_human, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(burja_2022_ctrl_human, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(burja_2022_ctrl_human) ** 0.5 // 2), metric='cosine')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(burja_2022_ctrl_human, min_dist=0.4, random_state=seed)\n",
    "sc.tl.leiden(burja_2022_ctrl_human, resolution=5, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.subsample(burja_2022_ctrl_human, fraction=1, random_state=0, copy=False)\n",
    "sc.pl.umap(burja_2022_ctrl_human, color=['leiden', 'Internal sample identifier'], legend_loc='on data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(burja_2022_ctrl_human, color=['leiden', 'LUM', 'PDGFRA', 'COL1A1', 'DCN', 'RGS5', 'MYH11', 'MLANA', 'PMEL', 'HBB', 'S100B'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(burja_2022_ctrl_human, dict_cats=dict_cats_fb_human, min_score=0.4, quantile_gene_sel=0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(burja_2022_ctrl_human, color=['leiden', 'assigned_cats'] + dict_cats_fb_human['peri - CYCS'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_fb_human.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(burja_2022_ctrl_human, color=['assigned_cats'] + [i for i in val if i in burja_2022_ctrl_human.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(burja_2022_ctrl_human, color=['leiden', 'COL18A1', 'APCDD1', 'SLPI', 'CCL19', 'GGT5',  'CHRDL1', 'GPX3', 'BGN', 'ASPN', 'TNN', 'COL11A1', 'COCH', \n",
    "                                     'IGFBP3', 'ANGPTL7', 'SCN7A', 'C2orf40', 'NGFR', 'CLDN1', 'SBSPON', 'FGFBP2', 'DIO3', 'LUZP2'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "burja_2022_ctrl_human_fb = burja_2022_ctrl_human[burja_2022_ctrl_human.obs['assigned_cats'].isin(['fibro'])].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(burja_2022_ctrl_human_fb, min_counts=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(burja_2022_ctrl_human_fb, random_state=seed, n_comps=50)\n",
    "sce.pp.bbknn(burja_2022_ctrl_human_fb, batch_key='Internal sample identifier')\n",
    "tk.tl.triku(burja_2022_ctrl_human_fb, use_raw=False)\n",
    "\n",
    "sc.pp.pca(burja_2022_ctrl_human_fb, random_state=seed, n_comps=50)\n",
    "sce.pp.bbknn(burja_2022_ctrl_human_fb, batch_key='Internal sample identifier')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(burja_2022_ctrl_human_fb, min_dist=0.25, random_state=seed)\n",
    "sc.tl.leiden(burja_2022_ctrl_human_fb, resolution=3, random_state=seed)  # leiden 9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(burja_2022_ctrl_human_fb, dict_cats=dict_cats_clusters, min_score=0.4, quantile_gene_sel=0.9, key_added='cluster', others_name='U')\n",
    "assign_cats(burja_2022_ctrl_human_fb, column_groupby='cluster', dict_cats=dict_cats_axes, min_score=0.5, quantile_gene_sel=0.9,  key_added='axis', \n",
    "             intermediate_states=True, diff=0.15, others_name='U')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "burja_2022_ctrl_human_fb.uns['cluster_colors'] = [dict_colors_human[i] if i in dict_colors_human else '#bcbcbc' for \n",
    "                                      i in sorted(set(burja_2022_ctrl_human_fb.obs['cluster']))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(burja_2022_ctrl_human_fb, color=['Internal sample identifier', 'leiden', 'axis', 'cluster'], legend_loc='on data', cmap=magma, use_raw=False, ncols=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "del burja_2022_ctrl_human_fb.obs['C3']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_clusters.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(burja_2022_ctrl_human_fb, color=['cluster'] + [i for i in val if i in burja_2022_ctrl_human_fb.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot_score_graph(burja_2022_ctrl_human_fb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "clear_adata(burja_2022_ctrl_human_fb)\n",
    "burja_2022_ctrl_human_fb.write_h5ad(burja_2022_dir + '/burja_2022_ctrl_human_fb_processed.h5')\n",
    "burja_2022_ctrl_human.write_h5ad(burja_2022_dir + '/burja_2022_ctrl_human_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "burja_2022_ctrl_human_fb = sc.read(burja_2022_dir + '/burja_2022_ctrl_human_fb_processed.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**U** cluster *BNIP3*, *BNIP3L*, *HILPDA*, *ERO1A*, *ENO2* indicates a possible hypoxia state in these cells."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.rank_genes_groups(burja_2022_ctrl_human_fb, groupby='leiden', groups=['5'], reference='rest')\n",
    "sc.pl.rank_genes_groups_tracksplot(burja_2022_ctrl_human_fb, dendrogram=False, n_genes=200)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Capolupo et al. 2022 [rejected because fbs are cultured and have a completely different transcriptome]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "capolupo_2022_dir = data_dir + '/capolupo_2022'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "capolupo_2022_ctrl_human = sc.read(capolupo_2022_dir + '/capolupo_2022_ctrl_human.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "capolupo_2022_ctrl_human.var_names = [dict_rep[i] if i in dict_rep else i for i in capolupo_2022_ctrl_human.var_names]\n",
    "capolupo_2022_ctrl_human.var['gene_symbol'] = capolupo_2022_ctrl_human.var_names\n",
    "capolupo_2022_ctrl_human.var['Ensemble ID'] = [dict_biomart_human[i] if i in dict_biomart_human else 'NA' for i in capolupo_2022_ctrl_human.var_names]\n",
    "\n",
    "capolupo_2022_ctrl_human.raw = capolupo_2022_ctrl_human.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Basic QC filtering\n",
    "capolupo_2022_ctrl_human.var['mt'] = capolupo_2022_ctrl_human.var_names.str.startswith('MT-')  # annotate the group of mitochondrial genes as 'mt'\n",
    "sc.pp.calculate_qc_metrics(capolupo_2022_ctrl_human, qc_vars=['mt'], percent_top=None, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.violin(capolupo_2022_ctrl_human, ['n_genes_by_counts', 'total_counts', 'pct_counts_mt'],\n",
    "             jitter=0.4, multi_panel=True)\n",
    "\n",
    "sc.pl.scatter(capolupo_2022_ctrl_human, x='total_counts', y='pct_counts_mt')\n",
    "sc.pl.scatter(capolupo_2022_ctrl_human, x='total_counts', y='n_genes_by_counts')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1, 1, figsize=(10, 4))\n",
    "df = pd.DataFrame({'x': capolupo_2022_ctrl_human.obs['Internal sample identifier'], 'y': capolupo_2022_ctrl_human.obs['log1p_n_genes_by_counts']})\n",
    "sns.violinplot(x='x', y='y', data=df, ax=ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "capolupo_2022_ctrl_human = capolupo_2022_ctrl_human[(((capolupo_2022_ctrl_human.obs['Internal sample identifier'] == 'HC01') & (capolupo_2022_ctrl_human.obs.log1p_n_genes_by_counts < 9.5) & \n",
    "                                                (capolupo_2022_ctrl_human.obs.log1p_n_genes_by_counts > 7.5)) | \n",
    "                                              ((capolupo_2022_ctrl_human.obs['Internal sample identifier'] == 'HC02') & (capolupo_2022_ctrl_human.obs.log1p_n_genes_by_counts < 9.5) & \n",
    "                                                (capolupo_2022_ctrl_human.obs.log1p_n_genes_by_counts > 7.5)) \n",
    "                                             ).values, :]\n",
    "capolupo_2022_ctrl_human = capolupo_2022_ctrl_human[capolupo_2022_ctrl_human.obs.pct_counts_mt < 7, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(capolupo_2022_ctrl_human, min_counts=1)\n",
    "sc.pp.normalize_total(capolupo_2022_ctrl_human)\n",
    "sc.pp.log1p(capolupo_2022_ctrl_human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(capolupo_2022_ctrl_human, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(capolupo_2022_ctrl_human, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(capolupo_2022_ctrl_human, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(capolupo_2022_ctrl_human) ** 0.5 // 2), metric='cosine')\n",
    "tk.tl.triku(capolupo_2022_ctrl_human, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(capolupo_2022_ctrl_human, min_dist=0.4, random_state=seed)\n",
    "sc.tl.leiden(capolupo_2022_ctrl_human, resolution=5, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.subsample(capolupo_2022_ctrl_human, fraction=1, random_state=0, copy=False)\n",
    "sc.pl.umap(capolupo_2022_ctrl_human, color=['leiden', 'Internal sample identifier'], legend_loc='on data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(capolupo_2022_ctrl_human, color=['leiden', 'LUM', 'PDGFRA', 'COL1A1', 'DCN', 'RGS5', 'MYH11', 'MLANA', 'PMEL', 'HBB', 'S100B'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(capolupo_2022_ctrl_human, dict_cats=dict_cats_fb_human, min_score=0.4, quantile_gene_sel=0.6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(capolupo_2022_ctrl_human, color=['leiden', 'assigned_cats'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_fb_human.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(capolupo_2022_ctrl_human, color=['assigned_cats'] + [i for i in val if i in capolupo_2022_ctrl_human.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "capolupo_2022_ctrl_human_fb = capolupo_2022_ctrl_human[capolupo_2022_ctrl_human.obs['assigned_cats'].isin(['fibro', 'F', 'peri - CYCS'])].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(capolupo_2022_ctrl_human_fb, min_counts=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(capolupo_2022_ctrl_human_fb, random_state=seed, n_comps=50)\n",
    "sce.pp.harmony_integrate(capolupo_2022_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(capolupo_2022_ctrl_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(0.1 * len(capolupo_2022_ctrl_human_fb) ** 0.5), metric='cosine')\n",
    "tk.tl.triku(capolupo_2022_ctrl_human_fb, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(capolupo_2022_ctrl_human_fb, min_dist=0.25, random_state=seed)\n",
    "sc.tl.leiden(capolupo_2022_ctrl_human_fb, resolution=9, random_state=seed)  # leiden 9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(capolupo_2022_ctrl_human_fb, dict_cats=dict_cats_clusters, min_score=0.6, quantile_gene_sel=0.65, key_added='cluster', others_name='U')\n",
    "assign_cats(capolupo_2022_ctrl_human_fb, column_groupby='cluster', dict_cats=dict_cats_axes, min_score=0.5, quantile_gene_sel=0.9,  key_added='axis', \n",
    "             intermediate_states=True, diff=0.15, others_name='U')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "capolupo_2022_ctrl_human_fb.uns['cluster_colors'] = [dict_colors_human[i] if i in dict_colors_human else '#bcbcbc' for \n",
    "                                      i in sorted(set(capolupo_2022_ctrl_human_fb.obs['cluster']))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(capolupo_2022_ctrl_human_fb, color=['Internal sample identifier', 'leiden', 'axis', 'cluster'], legend_loc='on data', cmap=magma, use_raw=False, ncols=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "del capolupo_2022_ctrl_human_fb.obs['C3']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_clusters.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(capolupo_2022_ctrl_human_fb, color=['cluster'] + [i for i in val if i in capolupo_2022_ctrl_human_fb.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot_score_graph(capolupo_2022_ctrl_human_fb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "clear_adata(capolupo_2022_ctrl_human_fb)\n",
    "capolupo_2022_ctrl_human_fb.write_h5ad(capolupo_2022_dir + '/capolupo_2022_ctrl_human_fb_processed.h5')\n",
    "capolupo_2022_ctrl_human.write_h5ad(capolupo_2022_dir + '/capolupo_2022_ctrl_human_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "capolupo_2022_ctrl_human_fb = sc.read(capolupo_2022_dir + '/capolupo_2022_ctrl_human_processed.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Deng et al. 2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "deng_dir = data_dir + '/deng_2021'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "deng_2021_scar_human = sc.read(deng_dir + '/deng_2021_scar_human.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "deng_2021_scar_human.var_names = [dict_rep[i] if i in dict_rep else i for i in deng_2021_scar_human.var_names]\n",
    "deng_2021_scar_human.var['gene_symbol'] = deng_2021_scar_human.var_names\n",
    "deng_2021_scar_human.var['Ensemble ID'] = [dict_biomart_human[i] if i in dict_biomart_human else 'NA' for i in deng_2021_scar_human.var_names]\n",
    "\n",
    "deng_2021_scar_human.raw = deng_2021_scar_human.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Basic QC filtering\n",
    "deng_2021_scar_human.var['mt'] = deng_2021_scar_human.var_names.str.startswith('MT-')  # annotate the group of mitochondrial genes as 'mt'\n",
    "sc.pp.calculate_qc_metrics(deng_2021_scar_human, qc_vars=['mt'], percent_top=None, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.violin(deng_2021_scar_human, ['n_genes_by_counts', 'total_counts', 'pct_counts_mt'],\n",
    "             jitter=0.4, multi_panel=True)\n",
    "\n",
    "sc.pl.scatter(deng_2021_scar_human, x='total_counts', y='pct_counts_mt')\n",
    "sc.pl.scatter(deng_2021_scar_human, x='total_counts', y='n_genes_by_counts')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1, 1, figsize=(10, 4))\n",
    "df = pd.DataFrame({'x': deng_2021_scar_human.obs['Internal sample identifier'], 'y': deng_2021_scar_human.obs['log1p_n_genes_by_counts']})\n",
    "sns.violinplot(x='x', y='y', data=df, ax=ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "deng_2021_scar_human = deng_2021_scar_human[((deng_2021_scar_human.obs.n_genes_by_counts < 4500) & \n",
    "                                    (deng_2021_scar_human.obs.n_genes_by_counts > 1000)).values, :]\n",
    "deng_2021_scar_human = deng_2021_scar_human[deng_2021_scar_human.obs.pct_counts_mt < 18, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(deng_2021_scar_human, min_counts=1)\n",
    "sc.pp.normalize_total(deng_2021_scar_human)\n",
    "sc.pp.log1p(deng_2021_scar_human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(deng_2021_scar_human, random_state=seed, n_comps=50)\n",
    "sce.pp.harmony_integrate(deng_2021_scar_human, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(deng_2021_scar_human, use_rep='X_pca_harmony', n_neighbors=int(0.5 * len(deng_2021_scar_human) ** 0.5 // 2), metric='cosine')\n",
    "tk.tl.triku(deng_2021_scar_human, use_raw=False)\n",
    "\n",
    "sc.pp.pca(deng_2021_scar_human, random_state=seed, n_comps=50)\n",
    "sce.pp.harmony_integrate(deng_2021_scar_human, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(deng_2021_scar_human, use_rep='X_pca_harmony', n_neighbors=int(0.5 * len(deng_2021_scar_human) ** 0.5 // 2), metric='cosine')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(deng_2021_scar_human, min_dist=0.2, random_state=seed)\n",
    "sc.tl.leiden(deng_2021_scar_human, resolution=1.3, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.subsample(deng_2021_scar_human, fraction=1, random_state=0, copy=False)\n",
    "sc.pl.umap(deng_2021_scar_human, color=['leiden', 'Sample identifier'], legend_loc='on data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(deng_2021_scar_human, color=['leiden', 'LUM', 'PDGFRA', 'COL1A1', 'DCN', 'RGS5', 'KRT5', 'DMKN'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(deng_2021_scar_human, color=['leiden', 'C2orf40', 'CDH19', 'ANGPTL7','PLEKHB1','ENTPD2', \n",
    "                                   'SLC2A1', 'CLDN1', 'TNNT2', 'C19orf33', 'SFRP5', 'WNT6', ], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(deng_2021_scar_human, dict_cats=dict_cats_fb_human, quantile_gene_sel=0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_fb_human.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(deng_2021_scar_human, color=['assigned_cats'] + [i for i in val if i in deng_2021_scar_human.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(deng_2021_scar_human, color=['leiden', 'assigned_cats'] + dict_cats_fb_human['endo venule'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "deng_2021_scar_human_fb = deng_2021_scar_human[deng_2021_scar_human.obs['assigned_cats'].isin(['fibro', 'unassigned', 'fibro - ANGPTL7'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(deng_2021_scar_human_fb, min_counts=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(deng_2021_scar_human_fb, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(deng_2021_scar_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(deng_2021_scar_human_fb, use_rep='X_pca_harmony', n_neighbors=int(0.5 * len(deng_2021_scar_human_fb) ** 0.5 // 5), metric='cosine')\n",
    "tk.tl.triku(deng_2021_scar_human_fb, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(deng_2021_scar_human_fb, min_dist=0.25, random_state=seed)\n",
    "sc.tl.leiden(deng_2021_scar_human_fb, resolution=18, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(deng_2021_scar_human_fb, dict_cats=dict_cats_clusters, min_score=0.5, quantile_gene_sel=0.75, key_added='cluster', others_name='U')\n",
    "assign_cats(deng_2021_scar_human_fb, column_groupby='cluster', dict_cats=dict_cats_axes, min_score=0.4, key_added='axis', intermediate_states=True, diff=0.15, others_name='U')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "deng_2021_scar_human_fb.uns['cluster_colors'] = [dict_colors_human[i] if i in dict_colors_human else '#bcbcbc' for \n",
    "                                      i in sorted(set(deng_2021_scar_human_fb.obs['cluster']))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(deng_2021_scar_human_fb, color=['leiden', 'axis', 'cluster', 'Internal sample identifier'], legend_loc='on data', cmap=magma, use_raw=False, ncols=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "del deng_2021_scar_human_fb.obs['C3']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_clusters.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(deng_2021_scar_human_fb, color=['cluster'] + [i for i in val if i in deng_2021_scar_human_fb.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot_score_graph(deng_2021_scar_human_fb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "clear_adata(deng_2021_scar_human_fb)\n",
    "deng_2021_scar_human_fb.write_h5ad(deng_dir + '/deng_2021_scar_human_fb_processed.h5')\n",
    "deng_2021_scar_human.write_h5ad(deng_dir + '/deng_2021_scar_human_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "deng_2021_scar_human_fb = sc.read(deng_dir + '/deng_2021_scar_human_fb_processed.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Gao et al. 2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "gao_dir = data_dir + '/gao_2021'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "gao_2021_ctrl_human = sc.read(gao_dir + '/gao_2021_ctrl_human.h5')\n",
    "gao_2021_ctrl_human = gao_2021_ctrl_human[gao_2021_ctrl_human.obs['Internal sample identifier'].isin(['Ctrl1', 'Ctrl2', 'Ctrl3'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gao_2021_ctrl_human.var_names = [dict_rep[i] if i in dict_rep else i for i in gao_2021_ctrl_human.var_names]\n",
    "gao_2021_ctrl_human.var['gene_symbol'] = gao_2021_ctrl_human.var_names\n",
    "gao_2021_ctrl_human.var['Ensemble ID'] = [dict_biomart_human[i] if i in dict_biomart_human else 'NA' for i in gao_2021_ctrl_human.var_names]\n",
    "\n",
    "gao_2021_ctrl_human.raw = gao_2021_ctrl_human.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Basic QC filtering\n",
    "gao_2021_ctrl_human.var['mt'] = gao_2021_ctrl_human.var_names.str.startswith('MT-')  # annotate the group of mitochondrial genes as 'mt'\n",
    "sc.pp.calculate_qc_metrics(gao_2021_ctrl_human, qc_vars=['mt'], percent_top=None, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.violin(gao_2021_ctrl_human, ['n_genes_by_counts', 'total_counts', 'pct_counts_mt'],\n",
    "             jitter=0.4, multi_panel=True)\n",
    "\n",
    "sc.pl.scatter(gao_2021_ctrl_human, x='total_counts', y='pct_counts_mt')\n",
    "sc.pl.scatter(gao_2021_ctrl_human, x='total_counts', y='n_genes_by_counts')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gao_2021_ctrl_human = gao_2021_ctrl_human[((gao_2021_ctrl_human.obs.n_genes_by_counts < 7000) & \n",
    "                                    (gao_2021_ctrl_human.obs.n_genes_by_counts > 500)).values, :]\n",
    "gao_2021_ctrl_human = gao_2021_ctrl_human[gao_2021_ctrl_human.obs.pct_counts_mt < 40, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(gao_2021_ctrl_human, min_counts=1)\n",
    "sc.pp.normalize_total(gao_2021_ctrl_human)\n",
    "sc.pp.log1p(gao_2021_ctrl_human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc.pp.pca(gao_2021_ctrl_human, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(gao_2021_ctrl_human, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(gao_2021_ctrl_human, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(gao_2021_ctrl_human) ** 0.5 // 2), metric='cosine')\n",
    "tk.tl.triku(gao_2021_ctrl_human, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc.tl.umap(gao_2021_ctrl_human, min_dist=0.1, random_state=seed)\n",
    "sc.tl.leiden(gao_2021_ctrl_human, resolution=1.5, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(gao_2021_ctrl_human, color=['leiden', 'Internal sample identifier'], legend_loc='on data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc.pl.umap(gao_2021_ctrl_human, color=['leiden', 'LUM', 'PDGFRA', 'COL1A1', 'DCN'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(gao_2021_ctrl_human, dict_cats=dict_cats_fb_human, min_score=0.4,  quantile_gene_sel=0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_fb_human.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(gao_2021_ctrl_human, color=['assigned_cats'] + [i for i in val if i in gao_2021_ctrl_human.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(gao_2021_ctrl_human, color=['leiden', 'assigned_cats'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "gao_2021_ctrl_human_fb = gao_2021_ctrl_human[gao_2021_ctrl_human.obs['assigned_cats'] == 'fibro']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(gao_2021_ctrl_human_fb, min_counts=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc.pp.pca(gao_2021_ctrl_human_fb, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(gao_2021_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(gao_2021_ctrl_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(gao_2021_ctrl_human_fb) ** 0.5 // 2), metric='cosine')\n",
    "tk.tl.triku(gao_2021_ctrl_human_fb, use_raw=False)\n",
    "\n",
    "sc.pp.pca(gao_2021_ctrl_human_fb, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(gao_2021_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(gao_2021_ctrl_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(gao_2021_ctrl_human_fb) ** 0.5 // 2), metric='cosine')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(gao_2021_ctrl_human_fb, min_dist=0.3, random_state=seed)\n",
    "sc.tl.leiden(gao_2021_ctrl_human_fb, resolution=10, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(gao_2021_ctrl_human_fb, dict_cats=dict_cats_clusters, min_score=0.5, quantile_gene_sel=0.75, key_added='cluster', others_name='U')\n",
    "assign_cats(gao_2021_ctrl_human_fb, column_groupby='cluster', dict_cats=dict_cats_axes, min_score=0.25, quantile_gene_sel=0.6,\n",
    "            key_added='axis', intermediate_states=True, diff=0.15, others_name='U')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "gao_2021_ctrl_human_fb.uns['cluster_colors'] = [dict_colors_human[i] if i in dict_colors_human else '#bcbcbc' for \n",
    "                                      i in sorted(set(gao_2021_ctrl_human_fb.obs['cluster']))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(gao_2021_ctrl_human_fb, color=['Internal sample identifier', 'leiden', 'axis', 'cluster'], legend_loc='on data', \n",
    "           cmap=magma, use_raw=False, ncols=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "gao_2021_ctrl_human_fb = gao_2021_ctrl_human_fb[gao_2021_ctrl_human_fb.obs['cluster'] != 'Glial']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(gao_2021_ctrl_human_fb, min_counts=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(gao_2021_ctrl_human_fb, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(gao_2021_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(gao_2021_ctrl_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(gao_2021_ctrl_human_fb) ** 0.5 // 2), metric='cosine')\n",
    "tk.tl.triku(gao_2021_ctrl_human_fb, use_raw=False)\n",
    "\n",
    "sc.pp.pca(gao_2021_ctrl_human_fb, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(gao_2021_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(gao_2021_ctrl_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(gao_2021_ctrl_human_fb) ** 0.5 // 3), metric='cosine')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(gao_2021_ctrl_human_fb, min_dist=0.3, random_state=seed)\n",
    "sc.tl.leiden(gao_2021_ctrl_human_fb, resolution=5, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(gao_2021_ctrl_human_fb, dict_cats=dict_cats_clusters, min_score=0.5, quantile_gene_sel=0.75, key_added='cluster', others_name='U')\n",
    "assign_cats(gao_2021_ctrl_human_fb, column_groupby='cluster', dict_cats=dict_cats_axes, min_score=0.25, quantile_gene_sel=0.6,\n",
    "            key_added='axis', intermediate_states=True, diff=0.15, others_name='U')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "gao_2021_ctrl_human_fb.uns['cluster_colors'] = [dict_colors_human[i] if i in dict_colors_human else '#bcbcbc' for \n",
    "                                      i in sorted(set(gao_2021_ctrl_human_fb.obs['cluster']))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(gao_2021_ctrl_human_fb, color=['Internal sample identifier', 'leiden', 'axis', 'cluster'], legend_loc='on data', \n",
    "           cmap=magma, use_raw=False, ncols=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The unassigned cluster doesn't show any relevant DEGs, so it is just \"rubbish\"."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "del gao_2021_ctrl_human_fb.obs['C3']\n",
    "del gao_2021_ctrl_human_fb.obs['C2']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_clusters.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(gao_2021_ctrl_human_fb, color=['cluster'] + [i for i in val if i in gao_2021_ctrl_human_fb.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot_score_graph(gao_2021_ctrl_human_fb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "clear_adata(gao_2021_ctrl_human_fb)\n",
    "gao_2021_ctrl_human_fb.write_h5ad(gao_dir + '/gao_2021_ctrl_human_fb_processed.h5')\n",
    "gao_2021_ctrl_human.write_h5ad(gao_dir + '/gao_2021_ctrl_human_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "gao_2021_ctrl_human_fb = sc.read(gao_dir + '/gao_2021_ctrl_human_fb_processed.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Gaydosik et al. 2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "gaydosik_dir = data_dir + '/gaydosik_2020'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gaydosik_2020_ctrl_human = sc.read_h5ad(gaydosik_dir + '/gaydosik_2020_ctrl_human.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gaydosik_2020_ctrl_human.var_names = [dict_rep[i] if i in dict_rep else i for i in gaydosik_2020_ctrl_human.var_names]\n",
    "gaydosik_2020_ctrl_human.var['gene_symbol'] = gaydosik_2020_ctrl_human.var_names\n",
    "gaydosik_2020_ctrl_human.var['Ensemble ID'] = [dict_biomart_human[i] if i in dict_biomart_human else 'NA' for i in gaydosik_2020_ctrl_human.var_names]\n",
    "\n",
    "gaydosik_2020_ctrl_human.raw = gaydosik_2020_ctrl_human.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Basic QC filtering\n",
    "gaydosik_2020_ctrl_human.var['mt'] = gaydosik_2020_ctrl_human.var_names.str.startswith('MT-')  # annotate the group of mitochondrial genes as 'mt'\n",
    "sc.pp.calculate_qc_metrics(gaydosik_2020_ctrl_human, qc_vars=['mt'], percent_top=None, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc.pl.violin(gaydosik_2020_ctrl_human, ['n_genes_by_counts', 'total_counts', 'pct_counts_mt'],\n",
    "             jitter=0.4, multi_panel=True)\n",
    "\n",
    "sc.pl.scatter(gaydosik_2020_ctrl_human, x='total_counts', y='pct_counts_mt')\n",
    "sc.pl.scatter(gaydosik_2020_ctrl_human, x='total_counts', y='n_genes_by_counts')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gaydosik_2020_ctrl_human = gaydosik_2020_ctrl_human[((gaydosik_2020_ctrl_human.obs.n_genes_by_counts < 5500) & \n",
    "                                    (gaydosik_2020_ctrl_human.obs.n_genes_by_counts > 400)).values, :]\n",
    "gaydosik_2020_ctrl_human = gaydosik_2020_ctrl_human[gaydosik_2020_ctrl_human.obs.pct_counts_mt < 30, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(gaydosik_2020_ctrl_human, min_counts=1)\n",
    "sc.pp.normalize_total(gaydosik_2020_ctrl_human)\n",
    "sc.pp.log1p(gaydosik_2020_ctrl_human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc.pp.pca(gaydosik_2020_ctrl_human, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(gaydosik_2020_ctrl_human, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(gaydosik_2020_ctrl_human, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(gaydosik_2020_ctrl_human) ** 0.5 // 2), metric='cosine')\n",
    "tk.tl.triku(gaydosik_2020_ctrl_human, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc.tl.umap(gaydosik_2020_ctrl_human, min_dist=0.1, random_state=seed)\n",
    "sc.tl.leiden(gaydosik_2020_ctrl_human, resolution=0.3, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(gaydosik_2020_ctrl_human, min_dist=0.1, random_state=seed)\n",
    "sc.tl.leiden(gaydosik_2020_ctrl_human, resolution=3, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(gaydosik_2020_ctrl_human, color=['leiden', 'Internal sample identifier'], legend_loc='on data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(gaydosik_2020_ctrl_human, color=['leiden', 'LUM', 'PDGFRA', 'COL1A1', 'DCN'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(gaydosik_2020_ctrl_human, dict_cats=dict_cats_fb_human, quantile_gene_sel=0.7,)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_fb_human.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(gaydosik_2020_ctrl_human, color=['assigned_cats'] + [i for i in val if i in gaydosik_2020_ctrl_human.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(gaydosik_2020_ctrl_human, color=['assigned_cats'] + ['KRT7', 'KRT8', 'KRT18', 'KRT19', 'PPARG'], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(gaydosik_2020_ctrl_human, color=['leiden', 'assigned_cats'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "gaydosik_2020_ctrl_human_fb = gaydosik_2020_ctrl_human[gaydosik_2020_ctrl_human.obs['assigned_cats'].isin(['fibro', 'fibro - ANGPTL7'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(gaydosik_2020_ctrl_human_fb, min_counts=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(gaydosik_2020_ctrl_human_fb, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(gaydosik_2020_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(gaydosik_2020_ctrl_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(gaydosik_2020_ctrl_human_fb) ** 0.5 // 2), metric='cosine')\n",
    "tk.tl.triku(gaydosik_2020_ctrl_human_fb, use_raw=False)\n",
    "\n",
    "sc.pp.pca(gaydosik_2020_ctrl_human_fb, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(gaydosik_2020_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(gaydosik_2020_ctrl_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(gaydosik_2020_ctrl_human_fb) ** 0.5 // 2), metric='cosine')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(gaydosik_2020_ctrl_human_fb, min_dist=0.25, random_state=seed)\n",
    "sc.tl.leiden(gaydosik_2020_ctrl_human_fb, resolution=3, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(gaydosik_2020_ctrl_human_fb, dict_cats=dict_cats_clusters, min_score=0.5, quantile_gene_sel=0.95, key_added='cluster', others_name='U')\n",
    "assign_cats(gaydosik_2020_ctrl_human_fb, column_groupby='cluster', dict_cats=dict_cats_axes, min_score=0.4, key_added='axis', \n",
    "             intermediate_states=True, diff=0.15, others_name='U')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "gaydosik_2020_ctrl_human_fb.uns['cluster_colors'] = [dict_colors_human[i] if i in dict_colors_human else '#bcbcbc' for \n",
    "                                      i in sorted(set(gaydosik_2020_ctrl_human_fb.obs['cluster']))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(gaydosik_2020_ctrl_human_fb, color=['Internal sample identifier', 'leiden', 'axis', 'cluster'], legend_loc='on data', \n",
    "           cmap=magma, use_raw=False, ncols=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "del gaydosik_2020_ctrl_human_fb.obs['C3']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_clusters.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(gaydosik_2020_ctrl_human_fb, color=['cluster'] + [i for i in val if i in gaydosik_2020_ctrl_human_fb.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot_score_graph(gaydosik_2020_ctrl_human_fb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "clear_adata(gaydosik_2020_ctrl_human_fb)\n",
    "gaydosik_2020_ctrl_human_fb.write_h5ad(gaydosik_dir + '/gaydosik_2020_ctrl_human_fb_processed.h5')\n",
    "gaydosik_2020_ctrl_human.write_h5ad(gaydosik_dir + '/gaydosik_2020_ctrl_human_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "gaydosik_2020_ctrl_human_fb = sc.read(gaydosik_dir + '/gaydosik_2020_ctrl_human_fb_processed.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Gur et al. 2022"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "gur_2022_dir = data_dir + '/gur_2022'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "gur_2022_ctrl_human = sc.read_h5ad(gur_2022_dir + '/gur_2022_ctrl_human.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gur_2022_ctrl_human.var_names = [dict_rep[i] if i in dict_rep else i for i in gur_2022_ctrl_human.var_names]\n",
    "gur_2022_ctrl_human.var['gene_symbol'] = gur_2022_ctrl_human.var_names\n",
    "gur_2022_ctrl_human.var['Ensemble ID'] = [dict_biomart_human[i] if i in dict_biomart_human else 'NA' for i in gur_2022_ctrl_human.var_names]\n",
    "\n",
    "gur_2022_ctrl_human.raw = gur_2022_ctrl_human.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Basic QC filtering\n",
    "gur_2022_ctrl_human.var['mt'] = gur_2022_ctrl_human.var_names.str.startswith('MT-')  # annotate the group of mitochondrial genes as 'mt'\n",
    "sc.pp.calculate_qc_metrics(gur_2022_ctrl_human, qc_vars=['mt'], percent_top=None, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.violin(gur_2022_ctrl_human, ['n_genes_by_counts', 'total_counts', 'pct_counts_mt'],\n",
    "             jitter=0.4, multi_panel=True)\n",
    "\n",
    "sc.pl.scatter(gur_2022_ctrl_human, x='total_counts', y='pct_counts_mt')\n",
    "sc.pl.scatter(gur_2022_ctrl_human, x='total_counts', y='n_genes_by_counts')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1, 1, figsize=(10, 4))\n",
    "df = pd.DataFrame({'x': gur_2022_ctrl_human.obs['Internal sample identifier'], 'y': gur_2022_ctrl_human.obs['log1p_n_genes_by_counts']})\n",
    "sns.violinplot(x='x', y='y', data=df, ax=ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "gur_2022_ctrl_human = gur_2022_ctrl_human[((gur_2022_ctrl_human.obs.n_genes_by_counts < 1950) & \n",
    "                                    (gur_2022_ctrl_human.obs.n_genes_by_counts > 300)).values, :]\n",
    "gur_2022_ctrl_human = gur_2022_ctrl_human[(gur_2022_ctrl_human.obs.pct_counts_mt < 40) & (gur_2022_ctrl_human.obs.pct_counts_mt > 10), :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(gur_2022_ctrl_human, min_counts=1)\n",
    "sc.pp.normalize_total(gur_2022_ctrl_human)\n",
    "sc.pp.log1p(gur_2022_ctrl_human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc.pp.pca(gur_2022_ctrl_human, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(gur_2022_ctrl_human, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(gur_2022_ctrl_human, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(gur_2022_ctrl_human) ** 0.5 // 2), metric='cosine')\n",
    "tk.tl.triku(gur_2022_ctrl_human, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(gur_2022_ctrl_human, min_dist=0.05, random_state=seed)\n",
    "sc.tl.leiden(gur_2022_ctrl_human, resolution=4,  random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(gur_2022_ctrl_human, color=['leiden', 'Internal sample identifier'], legend_loc='on data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(gur_2022_ctrl_human, color=['leiden', 'LUM', 'PDGFRA', 'COL1A1', 'DCN', 'KRT5', 'KRT14', 'C2orf40', 'ANGPTL7', 'SOX8', 'TIAM1',\n",
    "                                 'GPM6B','PLP1','S100B','SCN7A','NRXN1','GFRA3','MPZ', 'IGFBP2'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(gur_2022_ctrl_human, dict_cats=dict_cats_fb_human, quantile_gene_sel=0.8, min_score=0.45)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(gur_2022_ctrl_human, color=['leiden', 'assigned_cats', 'PDGFRA', 'LUM', 'DCN'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_fb_human.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(gur_2022_ctrl_human, color=['assigned_cats'] + [i for i in val if i in gur_2022_ctrl_human.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "gur_2022_ctrl_human_fb = gur_2022_ctrl_human[gur_2022_ctrl_human.obs['assigned_cats'].isin(['fibro', 'F', 'mt'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(gur_2022_ctrl_human_fb, min_counts=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(gur_2022_ctrl_human_fb, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(gur_2022_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(gur_2022_ctrl_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(gur_2022_ctrl_human_fb) ** 0.5 // 2), metric='cosine')\n",
    "tk.tl.triku(gur_2022_ctrl_human_fb, use_raw=False)\n",
    "\n",
    "sc.pp.pca(gur_2022_ctrl_human_fb, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(gur_2022_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(gur_2022_ctrl_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(gur_2022_ctrl_human_fb) ** 0.5 // 2), metric='cosine')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(gur_2022_ctrl_human_fb, min_dist=0.35, random_state=seed)\n",
    "sc.tl.leiden(gur_2022_ctrl_human_fb, resolution=3, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(gur_2022_ctrl_human_fb, dict_cats=dict_cats_clusters, min_score=0.5, quantile_gene_sel=0.9, key_added='cluster', others_name='U')\n",
    "assign_cats(gur_2022_ctrl_human_fb, column_groupby='cluster', dict_cats=dict_cats_axes, min_score=0.4, key_added='axis', \n",
    "             intermediate_states=True, diff=0.15, others_name='U')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "gur_2022_ctrl_human_fb.uns['cluster_colors'] = [dict_colors_human[i] if i in dict_colors_human else '#bcbcbc' for \n",
    "                                      i in sorted(set(gur_2022_ctrl_human_fb.obs['cluster']))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(gur_2022_ctrl_human_fb, color=['Internal sample identifier', 'leiden', 'axis', 'cluster', 'log1p_n_genes_by_counts'], legend_loc='on data', \n",
    "           cmap=magma, use_raw=False, ncols=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "del gur_2022_ctrl_human_fb.obs['C3']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_clusters.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(gur_2022_ctrl_human_fb, color=['cluster'] + [i for i in val if i in gur_2022_ctrl_human_fb.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot_score_graph(gur_2022_ctrl_human_fb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "clear_adata(gur_2022_ctrl_human_fb)\n",
    "gur_2022_ctrl_human_fb.write_h5ad(gur_2022_dir + '/gur_2022_ctrl_human_fb_processed.h5')\n",
    "gur_2022_ctrl_human.write_h5ad(gur_2022_dir + '/gur_2022_ctrl_human_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "gur_2022_ctrl_human_fb = sc.read(gur_2022_dir + '/gur_2022_ctrl_human_fb_processed.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### He et al. 2020 [Discarded becuase of bad quality cells]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "he_dir = data_dir + '/He_2020'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "he_2020_ctrl_human = sc.read_h5ad(he_dir + '/adata_he_2020_ctrl_human.h5')\n",
    "he_2020_ctrl_human.var_names_make_unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Replace CCN5 by WISP2 because it is a key gene\n",
    "he_2020_ctrl_human.var_names = [dict_rep[i] if i in dict_rep else i for i in he_2020_ctrl_human.var_names]\n",
    "he_2020_ctrl_human.var['gene_symbol'] = he_2020_ctrl_human.var_names\n",
    "he_2020_ctrl_human.var['Ensemble ID'] = [dict_biomart_human[i] if i in dict_biomart_human else 'NA' for i in he_2020_ctrl_human.var_names]\n",
    "\n",
    "he_2020_ctrl_human.raw = he_2020_ctrl_human.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Basic QC filtering\n",
    "he_2020_ctrl_human.var['mt'] = he_2020_ctrl_human.var_names.str.startswith('MT-')  # annotate the group of mitochondrial genes as 'mt'\n",
    "sc.pp.calculate_qc_metrics(he_2020_ctrl_human, qc_vars=['mt'], percent_top=None, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.violin(he_2020_ctrl_human, ['n_genes_by_counts', 'total_counts', 'pct_counts_mt'], jitter=0.4, multi_panel=True)\n",
    "sc.pl.scatter(he_2020_ctrl_human, x='total_counts', y='pct_counts_mt')\n",
    "sc.pl.scatter(he_2020_ctrl_human, x='total_counts', y='n_genes_by_counts')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "he_2020_ctrl_human = he_2020_ctrl_human[he_2020_ctrl_human.obs.n_genes_by_counts < 5000, :]\n",
    "he_2020_ctrl_human = he_2020_ctrl_human[he_2020_ctrl_human.obs.n_genes_by_counts > 600, :]\n",
    "he_2020_ctrl_human = he_2020_ctrl_human[he_2020_ctrl_human.obs.pct_counts_mt < 30, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(he_2020_ctrl_human, min_counts=1)\n",
    "sc.pp.normalize_total(he_2020_ctrl_human)\n",
    "sc.pp.log1p(he_2020_ctrl_human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(he_2020_ctrl_human, random_state=seed, n_comps=30)\n",
    "sc.pp.neighbors(he_2020_ctrl_human, random_state=seed, n_neighbors=int(0.5 * len(he_2020_ctrl_human) ** 0.5 // 4), metric='cosine')\n",
    "tk.tl.triku(he_2020_ctrl_human, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(he_2020_ctrl_human, min_dist=0.3, random_state=seed)\n",
    "sc.tl.leiden(he_2020_ctrl_human, resolution=3, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(he_2020_ctrl_human, color=['leiden', 'LUM', 'PDGFRA', 'COL1A1', 'DCN'], legend_loc='on data', cmap=magma, use_raw=False, ncols=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(he_2020_ctrl_human, dict_cats=dict_cats_fb_human, min_score=0.3, quantile_gene_sel=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_fb_human.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(he_2020_ctrl_human, color=['assigned_cats'] + [i for i in val if i in he_2020_ctrl_human.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(he_2020_ctrl_human, color=['leiden', 'assigned_cats', 'CDH19', 'ANGPTL7', 'PLEKHB1', 'ENTPD2', 'C2orf40', \n",
    "                           'SLC2A1', 'CLDN1', 'TNNT2', 'C19orf33', 'SFRP5'], legend_loc='on data', cmap=magma, use_raw=False, ncols=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(he_2020_ctrl_human, color=['leiden', 'assigned_cats'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "he_2020_ctrl_human_fb = he_2020_ctrl_human[he_2020_ctrl_human.obs['assigned_cats'].isin(['fibro', 'neuro', 'muscle', 'fibro - ANGPTL7'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(he_2020_ctrl_human_fb, min_counts=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(he_2020_ctrl_human_fb, random_state=seed, n_comps=50)\n",
    "sc.pp.neighbors(he_2020_ctrl_human_fb, random_state=seed, n_neighbors=int(0.5 * len(he_2020_ctrl_human_fb) ** 0.5 // 2), metric='cosine')\n",
    "tk.tl.triku(he_2020_ctrl_human_fb, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(he_2020_ctrl_human_fb, min_dist=0.2, random_state=seed)\n",
    "sc.tl.leiden(he_2020_ctrl_human_fb, resolution=15, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(he_2020_ctrl_human_fb, dict_cats=dict_cats_clusters, min_score=0.5, quantile_gene_sel=0.95, key_added='cluster')\n",
    "assign_cats(he_2020_ctrl_human_fb, column_groupby='cluster', dict_cats=dict_cats_axes, min_score=0.35, key_added='axis', \n",
    "            quantile_gene_sel=0.4, intermediate_states=True, diff=0.15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "he_2020_ctrl_human_fb.uns['cluster_colors'] = [dict_colors_human[i] if i in dict_colors_human else '#bcbcbc' for \n",
    "                                      i in sorted(set(he_2020_ctrl_human_fb.obs['cluster']))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(he_2020_ctrl_human_fb, color=['leiden', 'axis', 'cluster'], legend_loc='on data', \n",
    "           cmap=magma, use_raw=False, )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(he_2020_ctrl_human_fb, color=['DMKN', 'KRT5', 'KRT14'], legend_loc='on data', \n",
    "           cmap=magma, use_raw=False, )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "he_2020_ctrl_human_fb = he_2020_ctrl_human_fb[~ he_2020_ctrl_human_fb.obs['cluster'].isin(['Glial', 'unassigned', 'endo'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(he_2020_ctrl_human_fb, min_counts=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(he_2020_ctrl_human_fb, random_state=seed, n_comps=50)\n",
    "sc.pp.neighbors(he_2020_ctrl_human_fb, random_state=seed, n_neighbors=int(0.5 * len(he_2020_ctrl_human_fb) ** 0.5 // 2), metric='cosine')\n",
    "tk.tl.triku(he_2020_ctrl_human_fb, use_raw=False)\n",
    "\n",
    "sc.pp.pca(he_2020_ctrl_human_fb, random_state=seed, n_comps=50)\n",
    "sc.pp.neighbors(he_2020_ctrl_human_fb, random_state=seed, n_neighbors=int(0.5 * len(he_2020_ctrl_human_fb) ** 0.5 // 2), metric='cosine')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(he_2020_ctrl_human_fb, min_dist=0.4, random_state=seed)\n",
    "sc.tl.leiden(he_2020_ctrl_human_fb, resolution=7, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(he_2020_ctrl_human_fb, dict_cats=dict_cats_clusters, min_score=0.5, quantile_gene_sel=0.75, key_added='cluster', others_name='U')\n",
    "assign_cats(he_2020_ctrl_human_fb, column_groupby='cluster', dict_cats=dict_cats_axes, min_score=0.35, key_added='axis', \n",
    "            quantile_gene_sel=0.4, intermediate_states=True, diff=0.15, others_name='U')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "he_2020_ctrl_human_fb.uns['cluster_colors'] = [dict_colors_human[i] if i in dict_colors_human else '#bcbcbc' for \n",
    "                                      i in sorted(set(he_2020_ctrl_human_fb.obs['cluster']))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(he_2020_ctrl_human_fb, color=['leiden', 'axis', 'cluster'], legend_loc='on data', \n",
    "           cmap=magma, use_raw=False, )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "del he_2020_ctrl_human_fb.obs['C3']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.rank_genes_groups(he_2020_ctrl_human_fb, groupby='leiden', groups=['1', '2', '3'])\n",
    "sc.pl.rank_genes_groups_tracksplot(he_2020_ctrl_human_fb, dendrogram=False, n_genes=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_clusters.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(he_2020_ctrl_human_fb, color=['cluster'] + [i for i in val if i in he_2020_ctrl_human_fb.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot_score_graph(he_2020_ctrl_human_fb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "clear_adata(he_2020_ctrl_human_fb)\n",
    "he_2020_ctrl_human_fb.write_h5ad(he_dir + '/he_2020_ctrl_human_fb_processed.h5')\n",
    "he_2020_ctrl_human.write_h5ad(he_dir + '/he_2020_ctrl_human_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "he_2020_ctrl_human_fb = sc.read(he_dir + '/he_2020_ctrl_human_fb_processed.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Hughes et al. 2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "hughes_dir = data_dir + '/hughes_2020'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "hughes_2020_all = sc.read(hughes_dir + '/adata_hughes_2020_all.h5')\n",
    "hughes_2020_ctrl_human = hughes_2020_all[hughes_2020_all.obs['Internal sample identifier'].isin(['Normal1', 'Normal2', 'Normal3'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hughes_2020_ctrl_human.var_names = [dict_rep[i] if i in dict_rep else i for i in hughes_2020_ctrl_human.var_names]\n",
    "hughes_2020_ctrl_human.var['gene_symbol'] = hughes_2020_ctrl_human.var_names\n",
    "hughes_2020_ctrl_human.var['Ensemble ID'] = [dict_biomart_human[i] if i in dict_biomart_human else 'NA' for i in hughes_2020_ctrl_human.var_names]\n",
    "\n",
    "hughes_2020_ctrl_human.raw = hughes_2020_ctrl_human.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(hughes_2020_ctrl_human, min_counts=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Basic QC filtering\n",
    "hughes_2020_ctrl_human.var['mt'] = hughes_2020_ctrl_human.var_names.str.startswith('MT-')  # annotate the group of mitochondrial genes as 'mt'\n",
    "sc.pp.calculate_qc_metrics(hughes_2020_ctrl_human, qc_vars=['mt'], percent_top=None, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.violin(hughes_2020_ctrl_human, ['n_genes_by_counts', 'total_counts', 'pct_counts_mt'],\n",
    "             jitter=0.4, multi_panel=True)\n",
    "\n",
    "sc.pl.scatter(hughes_2020_ctrl_human, x='total_counts', y='pct_counts_mt')\n",
    "sc.pl.scatter(hughes_2020_ctrl_human, x='total_counts', y='n_genes_by_counts')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "hughes_2020_ctrl_human = hughes_2020_ctrl_human[((hughes_2020_ctrl_human.obs.n_genes_by_counts < 3000) & \n",
    "                                    (hughes_2020_ctrl_human.obs.n_genes_by_counts > 200)).values, :]\n",
    "hughes_2020_ctrl_human = hughes_2020_ctrl_human[hughes_2020_ctrl_human.obs.pct_counts_mt < 15, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "batches = sorted(list(set(hughes_2020_ctrl_human.obs['Internal sample identifier'].values)))\n",
    "for batch in batches:\n",
    "    counts = hughes_2020_ctrl_human.obs['n_genes_by_counts'].loc[hughes_2020_ctrl_human.obs['Internal sample identifier'] == batch].values\n",
    "    sns.distplot(counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(hughes_2020_ctrl_human, min_counts=1)\n",
    "sc.pp.normalize_total(hughes_2020_ctrl_human)\n",
    "sc.pp.log1p(hughes_2020_ctrl_human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(hughes_2020_ctrl_human, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(hughes_2020_ctrl_human, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(hughes_2020_ctrl_human, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(hughes_2020_ctrl_human) ** 0.5 // 2), metric='cosine')\n",
    "tk.tl.triku(hughes_2020_ctrl_human, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(hughes_2020_ctrl_human, min_dist=0.1, random_state=seed)\n",
    "sc.tl.leiden(hughes_2020_ctrl_human, resolution=2.5, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(hughes_2020_ctrl_human, color=['leiden', 'Internal sample identifier'], legend_loc='on data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(hughes_2020_ctrl_human, color=['leiden', 'LUM', 'PDGFRA', 'COL1A1', 'DCN', 'RGS5', 'PMEL'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(hughes_2020_ctrl_human, dict_cats=dict_cats_fb_human, min_score=0.3, quantile_gene_sel=0.99)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_fb_human.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(hughes_2020_ctrl_human, color=['assigned_cats'] + [i for i in val if i in hughes_2020_ctrl_human.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(hughes_2020_ctrl_human, color=['leiden', 'assigned_cats'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "hughes_2020_ctrl_human_fb = hughes_2020_ctrl_human[hughes_2020_ctrl_human.obs['assigned_cats'].isin(['fibro', 'F'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(hughes_2020_ctrl_human_fb, min_counts=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(hughes_2020_ctrl_human_fb, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(hughes_2020_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(hughes_2020_ctrl_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(hughes_2020_ctrl_human_fb) ** 0.5 // 2), metric='cosine')\n",
    "tk.tl.triku(hughes_2020_ctrl_human_fb, use_raw=False)\n",
    "\n",
    "sc.pp.pca(hughes_2020_ctrl_human_fb, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(hughes_2020_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(hughes_2020_ctrl_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(hughes_2020_ctrl_human_fb) ** 0.5 // 2), metric='cosine')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(hughes_2020_ctrl_human_fb, min_dist=0.35, random_state=seed)\n",
    "sc.tl.leiden(hughes_2020_ctrl_human_fb, resolution=7, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(hughes_2020_ctrl_human_fb, dict_cats=dict_cats_clusters, min_score=0.4, quantile_gene_sel=0.55, key_added='cluster', others_name='U')\n",
    "assign_cats(hughes_2020_ctrl_human_fb, column_groupby='cluster', dict_cats=dict_cats_axes, min_score=0.4, key_added='axis', \n",
    "             intermediate_states=True, diff=0.15, others_name='U')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "hughes_2020_ctrl_human_fb.uns['cluster_colors'] = [dict_colors_human[i] if i in dict_colors_human else '#bcbcbc' for \n",
    "                                      i in sorted(set(hughes_2020_ctrl_human_fb.obs['cluster']))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(hughes_2020_ctrl_human_fb, color=['Internal sample identifier', 'leiden', 'axis', 'cluster'], legend_loc='on data', \n",
    "           cmap=magma, use_raw=False, ncols=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "del hughes_2020_ctrl_human_fb.obs['C3']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_clusters.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(hughes_2020_ctrl_human_fb, color=['cluster'] + [i for i in val if i in hughes_2020_ctrl_human_fb.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot_score_graph(hughes_2020_ctrl_human_fb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "clear_adata(hughes_2020_ctrl_human_fb)\n",
    "hughes_2020_ctrl_human_fb.write_h5ad(hughes_dir + '/hughes_2020_ctrl_human_fb_processed.h5')\n",
    "hughes_2020_ctrl_human.write_h5ad(hughes_dir + '/hughes_2020_ctrl_human_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "hughes_2020_ctrl_human_fb = sc.read(hughes_dir + '/hughes_2020_ctrl_human_fb_processed.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Kim et al. 2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "kim_dir_2020 = data_dir + '/Kim_2020'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "kim_2020_ctrl_human = sc.read(kim_dir_2020 + '/adata_kim_2020_ctrl_human.h5')\n",
    "kim_2020_ctrl_human.var_names_make_unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "kim_2020_ctrl_human.var_names = [dict_rep[i] if i in dict_rep else i for i in kim_2020_ctrl_human.var_names]\n",
    "kim_2020_ctrl_human.var['gene_symbol'] = kim_2020_ctrl_human.var_names\n",
    "kim_2020_ctrl_human.var['Ensemble ID'] = [dict_biomart_human[i] if i in dict_biomart_human else 'NA' for i in kim_2020_ctrl_human.var_names]\n",
    "\n",
    "kim_2020_ctrl_human.raw = kim_2020_ctrl_human.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(kim_2020_ctrl_human, min_counts=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "kim_2020_ctrl_human.X = np.array(kim_2020_ctrl_human.X.todense())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Basic QC filtering\n",
    "kim_2020_ctrl_human.var['mt'] = kim_2020_ctrl_human.var_names.str.startswith('MT-')  # annotate the group of mitochondrial genes as 'mt'\n",
    "sc.pp.calculate_qc_metrics(kim_2020_ctrl_human, qc_vars=['mt'], percent_top=None, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.violin(kim_2020_ctrl_human, ['n_genes_by_counts', 'total_counts', 'pct_counts_mt'],\n",
    "             jitter=0.4, multi_panel=True)\n",
    "\n",
    "sc.pl.scatter(kim_2020_ctrl_human, x='total_counts', y='pct_counts_mt')\n",
    "sc.pl.scatter(kim_2020_ctrl_human, x='total_counts', y='n_genes_by_counts')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "kim_2020_ctrl_human = kim_2020_ctrl_human[((kim_2020_ctrl_human.obs.n_genes_by_counts < 4000) & \n",
    "                                    (kim_2020_ctrl_human.obs.n_genes_by_counts > 500)).values, :]\n",
    "kim_2020_ctrl_human = kim_2020_ctrl_human[kim_2020_ctrl_human.obs.pct_counts_mt < 25, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "batches = sorted(list(set(kim_2020_ctrl_human.obs['Internal sample identifier'].values)))\n",
    "for batch in batches:\n",
    "    counts = kim_2020_ctrl_human.obs['n_genes_by_counts'].loc[kim_2020_ctrl_human.obs['Internal sample identifier'] == batch].values\n",
    "    sns.distplot(counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(kim_2020_ctrl_human, min_counts=1)\n",
    "sc.pp.normalize_total(kim_2020_ctrl_human)\n",
    "sc.pp.log1p(kim_2020_ctrl_human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(kim_2020_ctrl_human, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(kim_2020_ctrl_human, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(kim_2020_ctrl_human, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(kim_2020_ctrl_human) ** 0.5 // 2), metric='cosine')\n",
    "tk.tl.triku(kim_2020_ctrl_human, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(kim_2020_ctrl_human, min_dist=0.1, random_state=seed)\n",
    "sc.tl.leiden(kim_2020_ctrl_human, resolution=0.7, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(kim_2020_ctrl_human, color=['leiden', 'Internal sample identifier'], legend_loc='on data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(kim_2020_ctrl_human, color=['leiden', 'LUM', 'PDGFRA', 'COL1A1', 'DCN', 'S100B', 'MPZ', 'DMKN', 'RGS5', 'C2orf40'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(kim_2020_ctrl_human, dict_cats=dict_cats_fb_human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(kim_2020_ctrl_human, color=['leiden', 'assigned_cats'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_fb_human.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(kim_2020_ctrl_human, color=['assigned_cats'] + [i for i in val if i in kim_2020_ctrl_human.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "kim_2020_ctrl_human_fb = kim_2020_ctrl_human[kim_2020_ctrl_human.obs['assigned_cats'] == 'fibro']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "np.unique(kim_2020_ctrl_human_fb.obs['Internal sample identifier'].values, return_counts=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "kim_2020_ctrl_human_fb = kim_2020_ctrl_human_fb[kim_2020_ctrl_human_fb.obs['Internal sample identifier'].isin(['HC1', 'HC2', 'HC3', 'HC4', 'HC6'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(kim_2020_ctrl_human_fb, min_counts=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(kim_2020_ctrl_human_fb, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(kim_2020_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(kim_2020_ctrl_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(kim_2020_ctrl_human_fb) ** 0.5 // 2), metric='cosine')\n",
    "tk.tl.triku(kim_2020_ctrl_human_fb, use_raw=False)\n",
    "\n",
    "sc.pp.pca(kim_2020_ctrl_human_fb, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(kim_2020_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(kim_2020_ctrl_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(kim_2020_ctrl_human_fb) ** 0.5 // 2), metric='cosine')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(kim_2020_ctrl_human_fb, min_dist=0.8, random_state=seed)\n",
    "sc.tl.leiden(kim_2020_ctrl_human_fb, resolution=10, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(kim_2020_ctrl_human_fb, dict_cats=dict_cats_clusters, min_score=0.5, quantile_gene_sel=0.75, key_added='cluster', others_name='U')\n",
    "assign_cats(kim_2020_ctrl_human_fb, column_groupby='cluster', dict_cats=dict_cats_axes, min_score=0.5, quantile_gene_sel=0.9, key_added='axis', \n",
    "             intermediate_states=True, diff=0.15, others_name='U')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "kim_2020_ctrl_human_fb.uns['cluster_colors'] = [dict_colors_human[i] if i in dict_colors_human else '#bcbcbc' for \n",
    "                                      i in sorted(set(kim_2020_ctrl_human_fb.obs['cluster']))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(kim_2020_ctrl_human_fb, color=['Internal sample identifier', 'leiden', 'axis', 'cluster'], legend_loc='on data', \n",
    "           cmap=magma, use_raw=False, ncols=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "del kim_2020_ctrl_human_fb.obs['C3']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_clusters.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(kim_2020_ctrl_human_fb, color=['cluster'] + [i for i in val if i in kim_2020_ctrl_human_fb.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot_score_graph(kim_2020_ctrl_human_fb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "clear_adata(kim_2020_ctrl_human_fb)\n",
    "kim_2020_ctrl_human_fb.write_h5ad(kim_dir_2020 + '/kim_2020_ctrl_human_fb_processed.h5')\n",
    "kim_2020_ctrl_human.write_h5ad(kim_dir_2020 + '/kim_2020_ctrl_human_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "kim_2020_ctrl_human_fb = sc.read(kim_dir_2020 + '/kim_2020_ctrl_human_fb_processed.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Kim et al. 2021 [NAIL, EXPECTING DIFFERENT RESULTS]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "kim_dir_2021 = data_dir + '/kim_2021'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "kim_2021_ctrl_human = sc.read(kim_dir_2021 + '/kim_2021_ctrl_human.h5')\n",
    "kim_2021_ctrl_human.var_names_make_unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "kim_2021_ctrl_human.var_names = [dict_rep[i] if i in dict_rep else i for i in kim_2021_ctrl_human.var_names]\n",
    "kim_2021_ctrl_human.var['gene_symbol'] = kim_2021_ctrl_human.var_names\n",
    "kim_2021_ctrl_human.var['Ensemble ID'] = [dict_biomart_human[i] if i in dict_biomart_human else 'NA' for i in kim_2021_ctrl_human.var_names]\n",
    "\n",
    "kim_2021_ctrl_human.raw = kim_2021_ctrl_human.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(kim_2021_ctrl_human, min_counts=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "kim_2021_ctrl_human.X = np.array(kim_2021_ctrl_human.X.todense())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Basic QC filtering\n",
    "kim_2021_ctrl_human.var['mt'] = kim_2021_ctrl_human.var_names.str.startswith('MT-')  # annotate the group of mitochondrial genes as 'mt'\n",
    "sc.pp.calculate_qc_metrics(kim_2021_ctrl_human, qc_vars=['mt'], percent_top=None, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.violin(kim_2021_ctrl_human, ['n_genes_by_counts', 'total_counts', 'pct_counts_mt'],\n",
    "             jitter=0.4, multi_panel=True)\n",
    "\n",
    "sc.pl.scatter(kim_2021_ctrl_human, x='total_counts', y='pct_counts_mt')\n",
    "sc.pl.scatter(kim_2021_ctrl_human, x='total_counts', y='n_genes_by_counts')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "kikim_2021_ctrl_humanm = kim_2021_ctrl_human[((kim_2021_ctrl_human.obs.n_genes_by_counts < 6000) & \n",
    "                                    (kim_2021_ctrl_human.obs.n_genes_by_counts > 1000)).values, :]\n",
    "kim_2021_ctrl_human = kim_2021_ctrl_human[kim_2021_ctrl_human.obs.pct_counts_mt < 40, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "kim_2021_ctrl_human"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "batches = sorted(list(set(kim_2021_ctrl_human.obs['Internal sample identifier'].values)))\n",
    "for batch in batches:\n",
    "    counts = kim_2021_ctrl_human.obs['n_genes_by_counts'].loc[kim_2021_ctrl_human.obs['Internal sample identifier'] == batch].values\n",
    "    sns.distplot(counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(kim_2021_ctrl_human, min_counts=1)\n",
    "sc.pp.normalize_total(kim_2021_ctrl_human)\n",
    "sc.pp.log1p(kim_2021_ctrl_human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(kim_2021_ctrl_human, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(kim_2021_ctrl_human, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(kim_2021_ctrl_human, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(kim_2021_ctrl_human) ** 0.5 // 2), metric='cosine')\n",
    "tk.tl.triku(kim_2021_ctrl_human, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(kim_2021_ctrl_human, min_dist=0.1, random_state=seed)\n",
    "sc.tl.leiden(kim_2021_ctrl_human, resolution=3, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.subsample(kim_2021_ctrl_human, fraction=1, random_state=0, copy=False)\n",
    "sc.pl.umap(kim_2021_ctrl_human, color=['leiden', 'Internal sample identifier'], legend_loc='on data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(kim_2021_ctrl_human, color=['leiden', 'LUM', 'PDGFRA', 'COL1A1', 'DCN'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(kim_2021_ctrl_human, dict_cats=dict_cats_fb_human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(kim_2021_ctrl_human, color=['leiden', 'assigned_cats'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(kim_2021_ctrl_human, color=['leiden', 'COL18A1', 'APCDD1', 'SLPI'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "kim_2021_ctrl_human_fb = kim_2021_ctrl_human[kim_2021_ctrl_human.obs['assigned_cats'] == 'fibro']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(kim_2021_ctrl_human_fb, min_counts=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(kim_2021_ctrl_human_fb, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(kim_2021_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(kim_2021_ctrl_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(kim_2021_ctrl_human_fb) ** 0.5 // 2), metric='cosine')\n",
    "tk.tl.triku(kim_2021_ctrl_human_fb, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(kim_2021_ctrl_human_fb, min_dist=0.4, random_state=seed)\n",
    "sc.tl.leiden(kim_2021_ctrl_human_fb, resolution=9, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(kim_2021_ctrl_human_fb, dict_cats=dict_cats_clusters, min_score=0.5, quantile_gene_sel=0.95, key_added='cluster', others_name='U')\n",
    "assign_cats(kim_2021_ctrl_human_fb, column_groupby='cluster', dict_cats=dict_cats_axes, min_score=0.35, key_added='axis', \n",
    "             intermediate_states=True, diff=0.15, others_name='U')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "kim_2021_ctrl_human_fb.uns['cluster_colors'] = [dict_colors_human[i] if i in dict_colors_human else '#bcbcbc' for \n",
    "                                      i in sorted(set(kim_2021_ctrl_human_fb.obs['cluster']))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(kim_2021_ctrl_human_fb, color=['Internal sample identifier', 'leiden', 'axis', 'cluster'], legend_loc='on data', cmap=magma, use_raw=False, ncols=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_clusters.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(kim_2021_ctrl_human_fb, color=['cluster'] + [i for i in val if i in kim_2021_ctrl_human_fb.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot_score_graph(kim_2021_ctrl_human_fb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "clear_adata(kim_2021_ctrl_human_fb)\n",
    "kim_2021_ctrl_human_fb.write_h5ad(kim_dir_2021 + '/kim_2021_ctrl_human_fb_processed.h5')\n",
    "kim_2021_ctrl_human.write_h5ad(kim_dir_2021 + '/kim_2021_ctrl_human_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "kim_2021_ctrl_human_fb = sc.read(kim_dir_2021 + '/kim_2021_ctrl_human_fb_processed.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Liu et al. 2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "liu_dir = data_dir + '/liu_2021'\n",
    "os.makedirs(liu_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "liu_2021_ctrl_human = sc.read(liu_dir + '/adata_liu_2021_ctrl_human.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "liu_2021_ctrl_human.var_names = [dict_rep[i] if i in dict_rep else i for i in liu_2021_ctrl_human.var_names]\n",
    "liu_2021_ctrl_human.var['gene_symbol'] = liu_2021_ctrl_human.var_names\n",
    "liu_2021_ctrl_human.var['Ensemble ID'] = [dict_biomart_human[i] if i in dict_biomart_human else 'NA' for i in liu_2021_ctrl_human.var_names]\n",
    "\n",
    "liu_2021_ctrl_human.raw = liu_2021_ctrl_human.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Basic QC filtering\n",
    "liu_2021_ctrl_human.var['mt'] = liu_2021_ctrl_human.var_names.str.startswith('MT-')  # annotate the group of mitochondrial genes as 'mt'\n",
    "sc.pp.calculate_qc_metrics(liu_2021_ctrl_human, qc_vars=['mt'], percent_top=None, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.violin(liu_2021_ctrl_human, ['n_genes_by_counts', 'total_counts', 'pct_counts_mt'],\n",
    "             jitter=0.4, multi_panel=True)\n",
    "\n",
    "sc.pl.scatter(liu_2021_ctrl_human, x='total_counts', y='pct_counts_mt')\n",
    "sc.pl.scatter(liu_2021_ctrl_human, x='total_counts', y='n_genes_by_counts')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1, 1, figsize=(10, 4))\n",
    "df = pd.DataFrame({'x': liu_2021_ctrl_human.obs['Internal sample identifier'], 'y': liu_2021_ctrl_human.obs['log1p_n_genes_by_counts']})\n",
    "sns.violinplot(x='x', y='y', data=df, ax=ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "liu_2021_ctrl_human = liu_2021_ctrl_human[(((liu_2021_ctrl_human.obs['Internal sample identifier'] == 'K007CTRL') & (liu_2021_ctrl_human.obs.log1p_n_genes_by_counts < 7.8) & \n",
    "                                                (liu_2021_ctrl_human.obs.log1p_n_genes_by_counts > 6.5)) | \n",
    "                                              ((liu_2021_ctrl_human.obs['Internal sample identifier'] == 'K009CTRL') & (liu_2021_ctrl_human.obs.log1p_n_genes_by_counts < 7.8) & \n",
    "                                                (liu_2021_ctrl_human.obs.log1p_n_genes_by_counts > 6.5)) | \n",
    "                                              ((liu_2021_ctrl_human.obs['Internal sample identifier'] == 'K012CTRL') & (liu_2021_ctrl_human.obs.log1p_n_genes_by_counts < 7.8) & \n",
    "                                                (liu_2021_ctrl_human.obs.log1p_n_genes_by_counts > 6.5)) | \n",
    "                                              ((liu_2021_ctrl_human.obs['Internal sample identifier'] == 'K013CTRL') & (liu_2021_ctrl_human.obs.log1p_n_genes_by_counts < 7.2) & \n",
    "                                                (liu_2021_ctrl_human.obs.log1p_n_genes_by_counts > 6))).values, :]\n",
    "liu_2021_ctrl_human = liu_2021_ctrl_human[liu_2021_ctrl_human.obs.pct_counts_mt < 15, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(liu_2021_ctrl_human, min_counts=1)\n",
    "sc.pp.normalize_total(liu_2021_ctrl_human)\n",
    "sc.pp.log1p(liu_2021_ctrl_human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(liu_2021_ctrl_human, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(liu_2021_ctrl_human, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(liu_2021_ctrl_human, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(liu_2021_ctrl_human) ** 0.5 // 2), metric='cosine')\n",
    "tk.tl.triku(liu_2021_ctrl_human, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(liu_2021_ctrl_human, min_dist=0.2, random_state=seed)\n",
    "sc.tl.leiden(liu_2021_ctrl_human, resolution=14, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.subsample(liu_2021_ctrl_human, fraction=1, random_state=0, copy=False)\n",
    "sc.pl.umap(liu_2021_ctrl_human, color=['leiden', 'Internal sample identifier'], legend_loc='on data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(liu_2021_ctrl_human, color=['leiden', 'LUM', 'PDGFRA', 'COL1A1', 'DCN', 'RGS5', 'MYH11', 'MLANA', 'PMEL'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(liu_2021_ctrl_human, dict_cats=dict_cats_fb_human, min_score=0.7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(liu_2021_ctrl_human, color=['leiden', 'assigned_cats'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_fb_human.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(liu_2021_ctrl_human, color=['assigned_cats'] + [i for i in val if i in liu_2021_ctrl_human.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(liu_2021_ctrl_human, color=['leiden', 'COL18A1', 'APCDD1', 'SLPI', 'CCL19'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "liu_2021_ctrl_human_fb = liu_2021_ctrl_human[liu_2021_ctrl_human.obs['assigned_cats'] == 'fibro']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(liu_2021_ctrl_human_fb, min_counts=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(liu_2021_ctrl_human_fb, random_state=seed, n_comps=50)\n",
    "sce.pp.harmony_integrate(liu_2021_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(liu_2021_ctrl_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(liu_2021_ctrl_human_fb) ** 0.5), metric='cosine')\n",
    "tk.tl.triku(liu_2021_ctrl_human_fb, use_raw=False)\n",
    "\n",
    "sc.pp.pca(liu_2021_ctrl_human_fb, random_state=seed, n_comps=50)\n",
    "sce.pp.harmony_integrate(liu_2021_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(liu_2021_ctrl_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(liu_2021_ctrl_human_fb) ** 0.5), metric='cosine')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(liu_2021_ctrl_human_fb, min_dist=0.4, random_state=seed)\n",
    "sc.tl.leiden(liu_2021_ctrl_human_fb, resolution=5, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(liu_2021_ctrl_human_fb, dict_cats=dict_cats_clusters, min_score=0.5, quantile_gene_sel=0.85, key_added='cluster', others_name='U')\n",
    "assign_cats(liu_2021_ctrl_human_fb, column_groupby='cluster', dict_cats=dict_cats_axes, min_score=0.5, quantile_gene_sel=0.9,  key_added='axis', \n",
    "             intermediate_states=True, diff=0.15, others_name='U')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "liu_2021_ctrl_human_fb.uns['cluster_colors'] = [dict_colors_human[i] if i in dict_colors_human else '#bcbcbc' for \n",
    "                                      i in sorted(set(liu_2021_ctrl_human_fb.obs['cluster']))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(liu_2021_ctrl_human_fb, color=['Internal sample identifier', 'leiden', 'axis', 'cluster'], legend_loc='on data', cmap=magma, use_raw=False, ncols=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "liu_2021_ctrl_human_fb = liu_2021_ctrl_human_fb[liu_2021_ctrl_human_fb.obs['cluster'] != 'Peri']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(liu_2021_ctrl_human_fb, min_counts=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(liu_2021_ctrl_human_fb, random_state=seed, n_comps=50)\n",
    "sce.pp.harmony_integrate(liu_2021_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(liu_2021_ctrl_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(liu_2021_ctrl_human_fb) ** 0.5), metric='cosine')\n",
    "tk.tl.triku(liu_2021_ctrl_human_fb, use_raw=False)\n",
    "\n",
    "sc.pp.pca(liu_2021_ctrl_human_fb, random_state=seed, n_comps=50)\n",
    "sce.pp.harmony_integrate(liu_2021_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(liu_2021_ctrl_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(liu_2021_ctrl_human_fb) ** 0.5), metric='cosine')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(liu_2021_ctrl_human_fb, min_dist=0.4, random_state=seed)\n",
    "sc.tl.leiden(liu_2021_ctrl_human_fb, resolution=5, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(liu_2021_ctrl_human_fb, dict_cats=dict_cats_clusters, min_score=0.5, quantile_gene_sel=0.85, key_added='cluster', others_name='U')\n",
    "assign_cats(liu_2021_ctrl_human_fb, column_groupby='cluster', dict_cats=dict_cats_axes, min_score=0.5, quantile_gene_sel=0.9,  key_added='axis', \n",
    "             intermediate_states=True, diff=0.15, others_name='U')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "liu_2021_ctrl_human_fb.uns['cluster_colors'] = [dict_colors_human[i] if i in dict_colors_human else '#bcbcbc' for \n",
    "                                      i in sorted(set(liu_2021_ctrl_human_fb.obs['cluster']))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(liu_2021_ctrl_human_fb, color=['Internal sample identifier', 'leiden', 'axis', 'cluster'], legend_loc='on data', cmap=magma, use_raw=False, ncols=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "del liu_2021_ctrl_human_fb.obs['C3']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_clusters.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(liu_2021_ctrl_human_fb, color=['cluster'] + [i for i in val if i in liu_2021_ctrl_human_fb.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot_score_graph(liu_2021_ctrl_human_fb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "clear_adata(liu_2021_ctrl_human_fb)\n",
    "liu_2021_ctrl_human_fb.write_h5ad(liu_dir + '/liu_2021_ctrl_human_fb_processed.h5')\n",
    "liu_2021_ctrl_human.write_h5ad(liu_dir + '/liu_2021_ctrl_human_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "liu_2021_ctrl_human_fb = sc.read(liu_dir + '/liu_2021_ctrl_human_fb_processed.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Mariottoni et al. 2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "mariottoni_2021_dir = data_dir + '/mariottoni_2021'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "mariottoni_2021_ctrl_human = sc.read(mariottoni_2021_dir + '/mariottoni_2021_ctrl_human.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "mariottoni_2021_ctrl_human.var_names = [dict_rep[i] if i in dict_rep else i for i in mariottoni_2021_ctrl_human.var_names]\n",
    "mariottoni_2021_ctrl_human.var['gene_symbol'] = mariottoni_2021_ctrl_human.var_names\n",
    "mariottoni_2021_ctrl_human.var['Ensemble ID'] = [dict_biomart_human[i] if i in dict_biomart_human else 'NA' for i in mariottoni_2021_ctrl_human.var_names]\n",
    "\n",
    "mariottoni_2021_ctrl_human.raw = mariottoni_2021_ctrl_human.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Basic QC filtering\n",
    "mariottoni_2021_ctrl_human.var['mt'] = mariottoni_2021_ctrl_human.var_names.str.startswith('MT-')  # annotate the group of mitochondrial genes as 'mt'\n",
    "sc.pp.calculate_qc_metrics(mariottoni_2021_ctrl_human, qc_vars=['mt'], percent_top=None, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.violin(mariottoni_2021_ctrl_human, ['n_genes_by_counts', 'total_counts', 'pct_counts_mt'],\n",
    "             jitter=0.4, multi_panel=True)\n",
    "\n",
    "sc.pl.scatter(mariottoni_2021_ctrl_human, x='total_counts', y='pct_counts_mt')\n",
    "sc.pl.scatter(mariottoni_2021_ctrl_human, x='total_counts', y='n_genes_by_counts')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1, 1, figsize=(10, 4))\n",
    "df = pd.DataFrame({'x': mariottoni_2021_ctrl_human.obs['Internal sample identifier'], 'y': mariottoni_2021_ctrl_human.obs['log1p_n_genes_by_counts']})\n",
    "sns.violinplot(x='x', y='y', data=df, ax=ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "mariottoni_2021_ctrl_human = mariottoni_2021_ctrl_human[(((mariottoni_2021_ctrl_human.obs['Internal sample identifier'] == 'HC') & (mariottoni_2021_ctrl_human.obs.log1p_n_genes_by_counts < 8.3) & \n",
    "                                                (mariottoni_2021_ctrl_human.obs.log1p_n_genes_by_counts > 6.3)) \n",
    "                                             ).values, :]\n",
    "mariottoni_2021_ctrl_human = mariottoni_2021_ctrl_human[mariottoni_2021_ctrl_human.obs.pct_counts_mt < 7, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(mariottoni_2021_ctrl_human, min_counts=1)\n",
    "sc.pp.normalize_total(mariottoni_2021_ctrl_human)\n",
    "sc.pp.log1p(mariottoni_2021_ctrl_human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(mariottoni_2021_ctrl_human, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(mariottoni_2021_ctrl_human, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(mariottoni_2021_ctrl_human, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(mariottoni_2021_ctrl_human) ** 0.5 // 2), metric='cosine')\n",
    "tk.tl.triku(mariottoni_2021_ctrl_human, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(mariottoni_2021_ctrl_human, min_dist=0.4, random_state=seed)\n",
    "sc.tl.leiden(mariottoni_2021_ctrl_human, resolution=5, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.subsample(mariottoni_2021_ctrl_human, fraction=1, random_state=0, copy=False)\n",
    "sc.pl.umap(mariottoni_2021_ctrl_human, color=['leiden', 'Internal sample identifier'], legend_loc='on data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(mariottoni_2021_ctrl_human, color=['leiden', 'LUM', 'PDGFRA', 'COL1A1', 'DCN', 'RGS5', 'MYH11', 'MLANA', 'PMEL', 'HBB', 'S100B', 'CLDN5', 'PECAM1'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(mariottoni_2021_ctrl_human, dict_cats=dict_cats_fb_human, min_score=0.45, quantile_gene_sel=0.65)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_fb_human.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(mariottoni_2021_ctrl_human, color=['assigned_cats'] + [i for i in val if i in mariottoni_2021_ctrl_human.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(mariottoni_2021_ctrl_human, color=['leiden', 'assigned_cats'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(mariottoni_2021_ctrl_human, color=['leiden', 'COL18A1', 'APCDD1', 'SLPI', 'CCL19', 'GGT5',  'CHRDL1', 'GPX3', 'BGN', 'ASPN', 'TNN', 'COL11A1', 'COCH', \n",
    "                                     'IGFBP3', 'ANGPTL7', 'SCN7A', 'C2orf40', 'NGFR', 'CLDN1', 'SBSPON', 'FGFBP2', 'DIO3', 'LUZP2'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "mariottoni_2021_ctrl_human_fb = mariottoni_2021_ctrl_human[mariottoni_2021_ctrl_human.obs['assigned_cats'].isin(['fibro'])].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(mariottoni_2021_ctrl_human_fb, min_counts=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(mariottoni_2021_ctrl_human_fb, random_state=seed, n_comps=50)\n",
    "sce.pp.harmony_integrate(mariottoni_2021_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(mariottoni_2021_ctrl_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(mariottoni_2021_ctrl_human_fb) ** 0.5), metric='cosine')\n",
    "tk.tl.triku(mariottoni_2021_ctrl_human_fb, use_raw=False)\n",
    "\n",
    "sc.pp.pca(mariottoni_2021_ctrl_human_fb, random_state=seed, n_comps=50)\n",
    "sce.pp.harmony_integrate(mariottoni_2021_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(mariottoni_2021_ctrl_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(mariottoni_2021_ctrl_human_fb) ** 0.5), metric='cosine')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(mariottoni_2021_ctrl_human_fb, min_dist=0.5, random_state=seed)\n",
    "sc.tl.leiden(mariottoni_2021_ctrl_human_fb, resolution=6, random_state=seed)  # leiden 9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(mariottoni_2021_ctrl_human_fb, dict_cats=dict_cats_clusters, min_score=0.6, quantile_gene_sel=0.9, key_added='cluster', others_name='U')\n",
    "assign_cats(mariottoni_2021_ctrl_human_fb, column_groupby='cluster', dict_cats=dict_cats_axes, min_score=0.5, quantile_gene_sel=0.9,  key_added='axis', \n",
    "             intermediate_states=True, diff=0.15, others_name='U')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "mariottoni_2021_ctrl_human_fb.uns['cluster_colors'] = [dict_colors_human[i] if i in dict_colors_human else '#bcbcbc' for \n",
    "                                      i in sorted(set(mariottoni_2021_ctrl_human_fb.obs['cluster']))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(mariottoni_2021_ctrl_human_fb, color=['Internal sample identifier', 'leiden', 'axis', 'cluster'], legend_loc='on data', cmap=magma, use_raw=False, ncols=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "mariottoni_2021_ctrl_human_fb = mariottoni_2021_ctrl_human_fb[~ mariottoni_2021_ctrl_human_fb.obs['cluster'].isin(['Glial', 'U', 'Endo'])].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(mariottoni_2021_ctrl_human_fb, min_counts=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(mariottoni_2021_ctrl_human_fb, random_state=seed, n_comps=50)\n",
    "sce.pp.harmony_integrate(mariottoni_2021_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(mariottoni_2021_ctrl_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(mariottoni_2021_ctrl_human_fb) ** 0.5), metric='cosine')\n",
    "tk.tl.triku(mariottoni_2021_ctrl_human_fb, use_raw=False)\n",
    "\n",
    "sc.pp.pca(mariottoni_2021_ctrl_human_fb, random_state=seed, n_comps=50)\n",
    "sce.pp.harmony_integrate(mariottoni_2021_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(mariottoni_2021_ctrl_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(mariottoni_2021_ctrl_human_fb) ** 0.5), metric='cosine')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(mariottoni_2021_ctrl_human_fb, min_dist=0.5, random_state=seed)\n",
    "sc.tl.leiden(mariottoni_2021_ctrl_human_fb, resolution=4, random_state=seed)  # leiden 9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(mariottoni_2021_ctrl_human_fb, dict_cats=dict_cats_clusters, min_score=0.6, quantile_gene_sel=0.9, key_added='cluster', others_name='U')\n",
    "assign_cats(mariottoni_2021_ctrl_human_fb, column_groupby='cluster', dict_cats=dict_cats_axes, min_score=0.5, quantile_gene_sel=0.9,  key_added='axis', \n",
    "             intermediate_states=True, diff=0.15, others_name='U')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "mariottoni_2021_ctrl_human_fb.uns['cluster_colors'] = [dict_colors_human[i] if i in dict_colors_human else '#bcbcbc' for \n",
    "                                      i in sorted(set(mariottoni_2021_ctrl_human_fb.obs['cluster']))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(mariottoni_2021_ctrl_human_fb, color=['Internal sample identifier', 'leiden', 'axis', 'cluster'], legend_loc='on data', cmap=magma, use_raw=False, ncols=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "del mariottoni_2021_ctrl_human_fb.obs['C3']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_clusters.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(mariottoni_2021_ctrl_human_fb, color=['cluster'] + [i for i in val if i in mariottoni_2021_ctrl_human_fb.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot_score_graph(mariottoni_2021_ctrl_human_fb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "clear_adata(mariottoni_2021_ctrl_human_fb)\n",
    "mariottoni_2021_ctrl_human_fb.write_h5ad(mariottoni_2021_dir + '/mariottoni_2021_ctrl_human_fb_processed.h5')\n",
    "mariottoni_2021_ctrl_human.write_h5ad(mariottoni_2021_dir + '/mariottoni_2021_ctrl_human_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "mariottoni_2021_ctrl_human_fb = sc.read(mariottoni_2021_dir + '/mariottoni_2021_ctrl_human_fb_processed.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Mirizio et al. 2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "mirizio_dir = data_dir + '/mirizio_2020'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "mirizio_2020_scl_human = sc.read(mirizio_dir + '/adata_mirizio_2020_scleroderma_human.h5')\n",
    "mirizio_2020_scl_human.var_names_make_unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mirizio_2020_scl_human.var_names = [dict_rep[i] if i in dict_rep else i for i in mirizio_2020_scl_human.var_names]\n",
    "mirizio_2020_scl_human.var['gene_symbol'] = mirizio_2020_scl_human.var_names\n",
    "mirizio_2020_scl_human.var['Ensemble ID'] = [dict_biomart_human[i] if i in dict_biomart_human else 'NA' for i in mirizio_2020_scl_human.var_names]\n",
    "\n",
    "mirizio_2020_scl_human.raw = mirizio_2020_scl_human.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Basic QC filtering\n",
    "mirizio_2020_scl_human.var['mt'] = mirizio_2020_scl_human.var_names.str.startswith('MT-')  # annotate the group of mitochondrial genes as 'mt'\n",
    "sc.pp.calculate_qc_metrics(mirizio_2020_scl_human, qc_vars=['mt'], percent_top=None, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.violin(mirizio_2020_scl_human, ['n_genes_by_counts', 'total_counts', 'pct_counts_mt'],\n",
    "             jitter=0.4, multi_panel=True)\n",
    "\n",
    "sc.pl.scatter(mirizio_2020_scl_human, x='total_counts', y='pct_counts_mt')\n",
    "sc.pl.scatter(mirizio_2020_scl_human, x='total_counts', y='n_genes_by_counts')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mirizio_2020_scl_human = mirizio_2020_scl_human[((mirizio_2020_scl_human.obs.n_genes_by_counts < 4000) & \n",
    "                                    (mirizio_2020_scl_human.obs.n_genes_by_counts > 250)).values, :]\n",
    "mirizio_2020_scl_human = mirizio_2020_scl_human[mirizio_2020_scl_human.obs.pct_counts_mt < 40, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(mirizio_2020_scl_human, min_counts=1)\n",
    "sc.pp.normalize_total(mirizio_2020_scl_human)\n",
    "sc.pp.log1p(mirizio_2020_scl_human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc.pp.pca(mirizio_2020_scl_human, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(mirizio_2020_scl_human, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(mirizio_2020_scl_human, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(mirizio_2020_scl_human) ** 0.5 // 2), metric='cosine')\n",
    "tk.tl.triku(mirizio_2020_scl_human, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc.tl.umap(mirizio_2020_scl_human, min_dist=0.1, random_state=seed)\n",
    "sc.tl.leiden(mirizio_2020_scl_human, resolution=7, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc.pl.umap(mirizio_2020_scl_human, color=['leiden', 'Internal sample identifier'], legend_loc='on data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc.pl.umap(mirizio_2020_scl_human, color=['leiden', 'LUM', 'PDGFRA', 'COL1A1', 'DCN'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assign_cats(mirizio_2020_scl_human, dict_cats=dict_cats_fb_human, min_score=0.4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(mirizio_2020_scl_human, color=['leiden', 'assigned_cats'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_fb_human.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(mirizio_2020_scl_human, color=['assigned_cats'] + [i for i in val if i in mirizio_2020_scl_human.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mirizio_2020_scl_human_fb = mirizio_2020_scl_human[mirizio_2020_scl_human.obs['assigned_cats'] == 'fibro']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(mirizio_2020_scl_human_fb, min_counts=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc.pp.pca(mirizio_2020_scl_human_fb, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(mirizio_2020_scl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(mirizio_2020_scl_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(mirizio_2020_scl_human_fb) ** 0.5 // 2), metric='cosine')\n",
    "tk.tl.triku(mirizio_2020_scl_human_fb, use_raw=False)\n",
    "\n",
    "sc.pp.pca(mirizio_2020_scl_human_fb, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(mirizio_2020_scl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(mirizio_2020_scl_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(mirizio_2020_scl_human_fb) ** 0.5 // 2), metric='cosine')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(mirizio_2020_scl_human_fb, min_dist=0.7, random_state=seed)\n",
    "sc.tl.leiden(mirizio_2020_scl_human_fb, resolution=8, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(mirizio_2020_scl_human_fb, dict_cats=dict_cats_clusters, min_score=0.45, quantile_gene_sel=0.75, key_added='cluster', others_name='U')\n",
    "assign_cats(mirizio_2020_scl_human_fb, column_groupby='cluster', dict_cats=dict_cats_axes, min_score=0.4, quantile_gene_sel=0.75,\n",
    "            key_added='axis', intermediate_states=True, diff=0.15, others_name='U')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "mirizio_2020_scl_human_fb.uns['cluster_colors'] = [dict_colors_human[i] if i in dict_colors_human else '#bcbcbc' for \n",
    "                                      i in sorted(set(mirizio_2020_scl_human_fb.obs['cluster']))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(mirizio_2020_scl_human_fb, color=['Internal sample identifier', 'leiden', 'axis', 'cluster'], legend_loc='on data', cmap=magma, use_raw=False, ncols=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "del mirizio_2020_scl_human_fb.obs['C3']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_clusters.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(mirizio_2020_scl_human_fb, color=['cluster'] + [i for i in val if i in mirizio_2020_scl_human_fb.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot_score_graph(mirizio_2020_scl_human_fb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "clear_adata(mirizio_2020_scl_human_fb)\n",
    "mirizio_2020_scl_human_fb.write_h5ad(mirizio_dir + '/mirizio_2020_scl_human_fb_processed.h5')\n",
    "mirizio_2020_scl_human.write_h5ad(mirizio_dir + '/mirizio_2020_scl_human_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "mirizio_2020_scl_human_fb = sc.read(mirizio_dir + '/mirizio_2020_scl_human_fb_processed.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reynolds et al. 2021 [Discarded because of bad quality cells]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "reynolds_dir = data_dir + '/reynolds_2021'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "reynolds_2021_ctrl_human_fb = sc.read(reynolds_dir + '/reynolds_2021_ctrl_human_fb.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reynolds_2021_ctrl_human_fb.var_names = [dict_rep[i] if i in dict_rep else i for i in reynolds_2021_ctrl_human_fb.var_names]\n",
    "reynolds_2021_ctrl_human_fb.var['gene_symbol'] = reynolds_2021_ctrl_human_fb.var_names\n",
    "reynolds_2021_ctrl_human_fb.var['Ensemble ID'] = [dict_biomart_human[i] if i in dict_biomart_human else 'NA' for i in reynolds_2021_ctrl_human_fb.var_names]\n",
    "\n",
    "reynolds_2021_ctrl_human_fb.raw.var['gene_symbol'] = reynolds_2021_ctrl_human_fb.raw.var_names\n",
    "reynolds_2021_ctrl_human_fb.raw.var['Ensemble ID'] = [dict_biomart_human[i] if i in dict_biomart_human else 'NA' for i in reynolds_2021_ctrl_human_fb.raw.var_names]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(reynolds_2021_ctrl_human_fb, color=['hypoxia_stress', 'Internal sample identifier', 'full_clustering'], legend_loc='on data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "reynolds_2021_ctrl_human_fb = reynolds_2021_ctrl_human_fb[reynolds_2021_ctrl_human_fb.obs['hypoxia_stress'] == 'Normal']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(reynolds_2021_ctrl_human_fb, min_counts=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(reynolds_2021_ctrl_human_fb, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(reynolds_2021_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(reynolds_2021_ctrl_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(reynolds_2021_ctrl_human_fb) ** 0.5 ), metric='cosine')\n",
    "tk.tl.triku(reynolds_2021_ctrl_human_fb, use_raw=False)\n",
    "\n",
    "sc.pp.pca(reynolds_2021_ctrl_human_fb, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(reynolds_2021_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(reynolds_2021_ctrl_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(reynolds_2021_ctrl_human_fb) ** 0.5 ), metric='cosine')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(reynolds_2021_ctrl_human_fb, min_dist=0.3, random_state=seed)\n",
    "sc.tl.leiden(reynolds_2021_ctrl_human_fb, resolution=7, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(reynolds_2021_ctrl_human_fb, dict_cats=dict_cats_clusters, min_score=0.5, quantile_gene_sel=0.9, key_added='cluster', others_name='U')\n",
    "assign_cats(reynolds_2021_ctrl_human_fb, column_groupby='cluster', dict_cats=dict_cats_axes, min_score=0.4, quantile_gene_sel=0.75,\n",
    "            key_added='axis', intermediate_states=True, diff=0.15, others_name='U')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "reynolds_2021_ctrl_human_fb.uns['cluster_colors'] = [dict_colors_human[i] if i in dict_colors_human else '#bcbcbc' for \n",
    "                                      i in sorted(set(reynolds_2021_ctrl_human_fb.obs['cluster']))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(reynolds_2021_ctrl_human_fb, color=['Internal sample identifier', 'leiden', 'axis', 'cluster'], legend_loc='on data', cmap=magma, use_raw=False, ncols=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(reynolds_2021_ctrl_human_fb, color=['COL18A1', 'CLEC2A', 'COL13A1', 'COL6A5', 'NPTX2', 'HSPB3', 'COMP', 'APCDD1', 'NKD2', 'AKAP6'], \n",
    "           legend_loc='on data', cmap=magma, use_raw=False, palette='Dark2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "del reynolds_2021_ctrl_human_fb.obs['C3']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_clusters.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(reynolds_2021_ctrl_human_fb, color=['cluster'] + [i for i in val if i in reynolds_2021_ctrl_human_fb.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot_score_graph(reynolds_2021_ctrl_human_fb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "clear_adata(reynolds_2021_ctrl_human_fb)\n",
    "reynolds_2021_ctrl_human_fb.write_h5ad(reynolds_dir + '/reynolds_2021_ctrl_human_fb_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "reynolds_2021_ctrl_human_fb = sc.read(reynolds_dir + '/reynolds_2021_ctrl_human_fb_processed.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Rindler et al. 2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "rindler_2021_dir = data_dir + '/rindler_2021'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "rindler_2021_ctrl_human = sc.read(rindler_2021_dir + '/rindler_2021_ctrl_human.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "rindler_2021_ctrl_human.var_names = [dict_rep[i] if i in dict_rep else i for i in rindler_2021_ctrl_human.var_names]\n",
    "rindler_2021_ctrl_human.var['gene_symbol'] = rindler_2021_ctrl_human.var_names\n",
    "rindler_2021_ctrl_human.var['Ensemble ID'] = [dict_biomart_human[i] if i in dict_biomart_human else 'NA' for i in rindler_2021_ctrl_human.var_names]\n",
    "\n",
    "rindler_2021_ctrl_human.raw = rindler_2021_ctrl_human.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Basic QC filtering\n",
    "rindler_2021_ctrl_human.var['mt'] = rindler_2021_ctrl_human.var_names.str.startswith('MT-')  # annotate the group of mitochondrial genes as 'mt'\n",
    "sc.pp.calculate_qc_metrics(rindler_2021_ctrl_human, qc_vars=['mt'], percent_top=None, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.violin(rindler_2021_ctrl_human, ['n_genes_by_counts', 'total_counts', 'pct_counts_mt'],\n",
    "             jitter=0.4, multi_panel=True)\n",
    "\n",
    "sc.pl.scatter(rindler_2021_ctrl_human, x='total_counts', y='pct_counts_mt')\n",
    "sc.pl.scatter(rindler_2021_ctrl_human, x='total_counts', y='n_genes_by_counts')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1, 1, figsize=(10, 4))\n",
    "df = pd.DataFrame({'x': rindler_2021_ctrl_human.obs['Internal sample identifier'], 'y': rindler_2021_ctrl_human.obs['log1p_n_genes_by_counts']})\n",
    "sns.violinplot(x='x', y='y', data=df, ax=ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "rindler_2021_ctrl_human = rindler_2021_ctrl_human[(((rindler_2021_ctrl_human.obs['Internal sample identifier'] == 'P112 HC') & (rindler_2021_ctrl_human.obs.log1p_n_genes_by_counts < 8) & \n",
    "                                                (rindler_2021_ctrl_human.obs.log1p_n_genes_by_counts > 6.8)) | \n",
    "                                              ((rindler_2021_ctrl_human.obs['Internal sample identifier'] == 'P115 HC') & (rindler_2021_ctrl_human.obs.log1p_n_genes_by_counts < 7.8) & \n",
    "                                                (rindler_2021_ctrl_human.obs.log1p_n_genes_by_counts > 6.8)) | \n",
    "                                              ((rindler_2021_ctrl_human.obs['Internal sample identifier'] == 'P116 HC') & (rindler_2021_ctrl_human.obs.log1p_n_genes_by_counts < 8.25) & \n",
    "                                                (rindler_2021_ctrl_human.obs.log1p_n_genes_by_counts > 7.3))| \n",
    "                                              ((rindler_2021_ctrl_human.obs['Internal sample identifier'] == 'P121 HC') & (rindler_2021_ctrl_human.obs.log1p_n_genes_by_counts < 7) & \n",
    "                                                (rindler_2021_ctrl_human.obs.log1p_n_genes_by_counts > 6.5)) \n",
    "                                             ).values, :]\n",
    "rindler_2021_ctrl_human = rindler_2021_ctrl_human[rindler_2021_ctrl_human.obs.pct_counts_mt < 7, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(rindler_2021_ctrl_human, min_counts=1)\n",
    "sc.pp.normalize_total(rindler_2021_ctrl_human)\n",
    "sc.pp.log1p(rindler_2021_ctrl_human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(rindler_2021_ctrl_human, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(rindler_2021_ctrl_human, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(rindler_2021_ctrl_human, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(rindler_2021_ctrl_human) ** 0.5 // 2), metric='cosine')\n",
    "tk.tl.triku(rindler_2021_ctrl_human, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(rindler_2021_ctrl_human, min_dist=0.4, random_state=seed)\n",
    "sc.tl.leiden(rindler_2021_ctrl_human, resolution=3, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.subsample(rindler_2021_ctrl_human, fraction=1, random_state=0, copy=False)\n",
    "sc.pl.umap(rindler_2021_ctrl_human, color=['leiden', 'Internal sample identifier'], legend_loc='on data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(rindler_2021_ctrl_human, color=['leiden', 'LUM', 'PDGFRA', 'COL1A1', 'DCN', 'RGS5', 'MYH11', 'MLANA', 'PMEL', 'HBB'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(rindler_2021_ctrl_human, dict_cats=dict_cats_fb_human, min_score=0.45, quantile_gene_sel=0.85)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(rindler_2021_ctrl_human, color=['leiden', 'assigned_cats'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_fb_human.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(rindler_2021_ctrl_human, color=['assigned_cats'] + [i for i in val if i in rindler_2021_ctrl_human.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(rindler_2021_ctrl_human, color=['leiden', 'COL18A1', 'APCDD1', 'SLPI', 'CCL19', 'GGT5',  'CHRDL1', 'GPX3', 'BGN', 'ASPN', 'TNN', 'COL11A1', 'COCH', \n",
    "                                     'IGFBP3', 'ANGPTL7', 'SCN7A', 'C2orf40', 'NGFR', 'CLDN1', 'SBSPON', 'FGFBP2', 'DIO3', 'LUZP2'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "rindler_2021_ctrl_human_fb = rindler_2021_ctrl_human[rindler_2021_ctrl_human.obs['assigned_cats'].isin(['fibro'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(rindler_2021_ctrl_human_fb, min_counts=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(rindler_2021_ctrl_human_fb, random_state=seed, n_comps=50)\n",
    "sce.pp.harmony_integrate(rindler_2021_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(rindler_2021_ctrl_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(0.1 * len(rindler_2021_ctrl_human_fb) ** 0.5), metric='cosine')\n",
    "tk.tl.triku(rindler_2021_ctrl_human_fb, use_raw=False)\n",
    "\n",
    "sc.pp.pca(rindler_2021_ctrl_human_fb, random_state=seed, n_comps=50)\n",
    "sce.pp.harmony_integrate(rindler_2021_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(rindler_2021_ctrl_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(0.1 * len(rindler_2021_ctrl_human_fb) ** 0.5), metric='cosine')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(rindler_2021_ctrl_human_fb, min_dist=0.5, random_state=seed)\n",
    "sc.tl.leiden(rindler_2021_ctrl_human_fb, resolution=15, random_state=seed) # 18"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(rindler_2021_ctrl_human_fb, dict_cats=dict_cats_clusters, min_score=0.45, quantile_gene_sel=0.65, key_added='cluster', others_name='U')\n",
    "assign_cats(rindler_2021_ctrl_human_fb, column_groupby='cluster', dict_cats=dict_cats_axes, min_score=0.5, quantile_gene_sel=0.9,  key_added='axis', \n",
    "             intermediate_states=True, diff=0.15, others_name='U')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "rindler_2021_ctrl_human_fb.uns['cluster_colors'] = [dict_colors_human[i] if i in dict_colors_human else '#bcbcbc' for \n",
    "                                      i in sorted(set(rindler_2021_ctrl_human_fb.obs['cluster']))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(rindler_2021_ctrl_human_fb, color=['Internal sample identifier', 'leiden', 'axis', 'cluster'], legend_loc='on data', cmap=magma, use_raw=False, ncols=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "rindler_2021_ctrl_human_fb = rindler_2021_ctrl_human_fb[~ rindler_2021_ctrl_human_fb.obs['cluster'].isin(['U', 'Endo'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(rindler_2021_ctrl_human_fb, min_counts=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(rindler_2021_ctrl_human_fb, random_state=seed, n_comps=50)\n",
    "sce.pp.harmony_integrate(rindler_2021_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(rindler_2021_ctrl_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(0.1 * len(rindler_2021_ctrl_human_fb) ** 0.5), metric='cosine')\n",
    "tk.tl.triku(rindler_2021_ctrl_human_fb, use_raw=False)\n",
    "\n",
    "sc.pp.pca(rindler_2021_ctrl_human_fb, random_state=seed, n_comps=50)\n",
    "sce.pp.harmony_integrate(rindler_2021_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(rindler_2021_ctrl_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(0.1 * len(rindler_2021_ctrl_human_fb) ** 0.5), metric='cosine')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(rindler_2021_ctrl_human_fb, min_dist=0.5, random_state=seed)\n",
    "sc.tl.leiden(rindler_2021_ctrl_human_fb, resolution=6, random_state=seed) # 18"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(rindler_2021_ctrl_human_fb, dict_cats=dict_cats_clusters, min_score=0.45, quantile_gene_sel=0.85, key_added='cluster', others_name='U')\n",
    "assign_cats(rindler_2021_ctrl_human_fb, column_groupby='cluster', dict_cats=dict_cats_axes, min_score=0.5, quantile_gene_sel=0.9,  key_added='axis', \n",
    "             intermediate_states=True, diff=0.15, others_name='U')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "rindler_2021_ctrl_human_fb.uns['cluster_colors'] = [dict_colors_human[i] if i in dict_colors_human else '#bcbcbc' for \n",
    "                                      i in sorted(set(rindler_2021_ctrl_human_fb.obs['cluster']))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(rindler_2021_ctrl_human_fb, color=['Internal sample identifier', 'leiden', 'axis', 'cluster'], legend_loc='on data', cmap=magma, use_raw=False, ncols=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-03-09T20:16:28.733418Z",
     "iopub.status.busy": "2022-03-09T20:16:28.733225Z",
     "iopub.status.idle": "2022-03-09T20:16:28.762323Z",
     "shell.execute_reply": "2022-03-09T20:16:28.761468Z",
     "shell.execute_reply.started": "2022-03-09T20:16:28.733392Z"
    }
   },
   "source": [
    "**MIRAR EL CLUSTER C5 Y DEGS DE B1/B2**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "del rindler_2021_ctrl_human_fb.obs['C3']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_clusters.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(rindler_2021_ctrl_human_fb, color=['cluster'] + [i for i in val if i in rindler_2021_ctrl_human_fb.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot_score_graph(rindler_2021_ctrl_human_fb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "clear_adata(rindler_2021_ctrl_human_fb)\n",
    "rindler_2021_ctrl_human_fb.write_h5ad(rindler_2021_dir + '/rindler_2021_ctrl_human_fb_processed.h5')\n",
    "rindler_2021_ctrl_human.write_h5ad(rindler_2021_dir + '/rindler_2021_ctrl_human_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "rindler_2021_ctrl_human_fb = sc.read(rindler_2021_dir + '/rindler_2021_ctrl_human_fb_processed.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Saluzzo et al. 2022 [Discarded because too few fibroblasts (~80)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "saluzzo_2022_dir = data_dir + '/saluzzo_2022'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "saluzzo_2022_ctrl_human = sc.read(saluzzo_2022_dir + '/saluzzo_2022_ctrl_human.h5')\n",
    "saluzzo_2022_ctrl_human.var_names_make_unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "saluzzo_2022_ctrl_human.var_names = [dict_rep[i] if i in dict_rep else i for i in saluzzo_2022_ctrl_human.var_names]\n",
    "saluzzo_2022_ctrl_human.var['gene_symbol'] = saluzzo_2022_ctrl_human.var_names\n",
    "saluzzo_2022_ctrl_human.var['Ensemble ID'] = [dict_biomart_human[i] if i in dict_biomart_human else 'NA' for i in saluzzo_2022_ctrl_human.var_names]\n",
    "\n",
    "saluzzo_2022_ctrl_human.raw = saluzzo_2022_ctrl_human.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(saluzzo_2022_ctrl_human, min_counts=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Basic QC filtering\n",
    "saluzzo_2022_ctrl_human.var['mt'] = saluzzo_2022_ctrl_human.var_names.str.startswith('MT-')  # annotate the group of mitochondrial genes as 'mt'\n",
    "sc.pp.calculate_qc_metrics(saluzzo_2022_ctrl_human, qc_vars=['mt'], percent_top=None, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.violin(saluzzo_2022_ctrl_human, ['n_genes_by_counts', 'total_counts', 'pct_counts_mt'],\n",
    "             jitter=0.4, multi_panel=True)\n",
    "\n",
    "sc.pl.scatter(saluzzo_2022_ctrl_human, x='total_counts', y='pct_counts_mt')\n",
    "sc.pl.scatter(saluzzo_2022_ctrl_human, x='total_counts', y='n_genes_by_counts')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1, 1, figsize=(10, 4))\n",
    "df = pd.DataFrame({'x': saluzzo_2022_ctrl_human.obs['Internal sample identifier'], 'y': saluzzo_2022_ctrl_human.obs['log1p_n_genes_by_counts']})\n",
    "sns.violinplot(x='x', y='y', data=df, ax=ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "saluzzo_2022_ctrl_human = saluzzo_2022_ctrl_human[(((saluzzo_2022_ctrl_human.obs['Internal sample identifier'] == 'H1') & (saluzzo_2022_ctrl_human.obs.log1p_n_genes_by_counts < 7.7) & \n",
    "                                                (saluzzo_2022_ctrl_human.obs.log1p_n_genes_by_counts > 6.8)) | \n",
    "                                              ((saluzzo_2022_ctrl_human.obs['Internal sample identifier'] == 'H2') & (saluzzo_2022_ctrl_human.obs.log1p_n_genes_by_counts < 8.1) & \n",
    "                                                (saluzzo_2022_ctrl_human.obs.log1p_n_genes_by_counts > 7)) | \n",
    "                                              ((saluzzo_2022_ctrl_human.obs['Internal sample identifier'] == 'H3') & (saluzzo_2022_ctrl_human.obs.log1p_n_genes_by_counts < 8.1) & \n",
    "                                                (saluzzo_2022_ctrl_human.obs.log1p_n_genes_by_counts > 7))\n",
    "                                             ).values, :]\n",
    "saluzzo_2022_ctrl_human = saluzzo_2022_ctrl_human[saluzzo_2022_ctrl_human.obs.pct_counts_mt < 10, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(saluzzo_2022_ctrl_human, min_counts=1)\n",
    "sc.pp.normalize_total(saluzzo_2022_ctrl_human)\n",
    "sc.pp.log1p(saluzzo_2022_ctrl_human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(saluzzo_2022_ctrl_human, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(saluzzo_2022_ctrl_human, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(saluzzo_2022_ctrl_human, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(saluzzo_2022_ctrl_human) ** 0.5 // 2), metric='cosine')\n",
    "tk.tl.triku(saluzzo_2022_ctrl_human, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(saluzzo_2022_ctrl_human, min_dist=0.6, random_state=seed)\n",
    "sc.tl.leiden(saluzzo_2022_ctrl_human, resolution=10, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(saluzzo_2022_ctrl_human, color=['Internal sample identifier', 'leiden', 'LUM', 'PDGFRA', 'COL1A1', 'DCN'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(saluzzo_2022_ctrl_human, dict_cats=dict_cats_fb_human, min_score=0.3, quantile_gene_sel=0.8) # 0,2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(saluzzo_2022_ctrl_human, color=['Internal sample identifier', 'leiden', 'assigned_cats'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_fb_human.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(saluzzo_2022_ctrl_human, color=['assigned_cats'] + [i for i in val if i in saluzzo_2022_ctrl_human.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "saluzzo_2022_ctrl_human_fb = saluzzo_2022_ctrl_human[saluzzo_2022_ctrl_human.obs['assigned_cats'].isin(['fibro', 'fibro - ANGPTL7'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(saluzzo_2022_ctrl_human_fb, min_counts=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(saluzzo_2022_ctrl_human_fb, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(saluzzo_2022_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(saluzzo_2022_ctrl_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(saluzzo_2022_ctrl_human_fb) ** 0.5 // 2), metric='cosine')\n",
    "tk.tl.triku(saluzzo_2022_ctrl_human_fb, use_raw=False)\n",
    "\n",
    "sc.pp.pca(saluzzo_2022_ctrl_human_fb, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(saluzzo_2022_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(saluzzo_2022_ctrl_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(saluzzo_2022_ctrl_human_fb) ** 0.5 // 2), metric='cosine')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(saluzzo_2022_ctrl_human_fb, min_dist=0.3, random_state=seed)\n",
    "sc.tl.leiden(saluzzo_2022_ctrl_human_fb, resolution=8, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(saluzzo_2022_ctrl_human_fb, dict_cats=dict_cats_clusters, min_score=0.5, quantile_gene_sel=0.9, key_added='cluster', others_name='U')\n",
    "assign_cats(saluzzo_2022_ctrl_human_fb, column_groupby='cluster', dict_cats=dict_cats_axes, min_score=0.5, quantile_gene_sel=0.9,  key_added='axis', \n",
    "            intermediate_states=True, diff=0.15, others_name='U')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "saluzzo_2022_ctrl_human_fb.uns['cluster_colors'] = [dict_colors_human[i] if i in dict_colors_human else '#bcbcbc' for \n",
    "                                      i in sorted(set(saluzzo_2022_ctrl_human_fb.obs['cluster']))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(saluzzo_2022_ctrl_human_fb, color=['Internal sample identifier', 'leiden', 'axis', 'cluster'], legend_loc='on data', cmap=magma, use_raw=False, ncols=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot_score_graph(saluzzo_2022_ctrl_human_fb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "del saluzzo_2022_ctrl_human_fb.obs['C3']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_clusters.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(saluzzo_2022_ctrl_human_fb, color=['cluster'] + [i for i in val if i in saluzzo_2022_ctrl_human_fb.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clear_adata(saluzzo_2022_ctrl_human_fb)\n",
    "saluzzo_2022_ctrl_human_fb.write_h5ad(saluzzo_2022_dir + '/saluzzo_2022_ctrl_human_fb_processed.h5')\n",
    "saluzzo_2022_ctrl_human.write_h5ad(saluzzo_2022_dir + '/saluzzo_2022_ctrl_human_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "saluzzo_2022_ctrl_human_fb = sc.read(saluzzo_2022_dir + '/saluzzo_2022_ctrl_human_fb_processed.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Solé-Boldo et al. 2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sole_dir = data_dir + '/Sole-Boldo_2020'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sole_2020_young_human = sc.read(sole_dir + '/adata_sole_2020_young_human.h5')\n",
    "sole_2020_young_human.var_names_make_unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sole_2020_young_human.var_names = [dict_rep[i] if i in dict_rep else i for i in sole_2020_young_human.var_names]\n",
    "sole_2020_young_human.var['gene_symbol'] = sole_2020_young_human.var_names\n",
    "sole_2020_young_human.var['Ensemble ID'] = [dict_biomart_human[i] if i in dict_biomart_human else 'NA' for i in sole_2020_young_human.var_names]\n",
    "\n",
    "sole_2020_young_human.raw = sole_2020_young_human.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(sole_2020_young_human, min_counts=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Basic QC filtering\n",
    "sole_2020_young_human.var['mt'] = sole_2020_young_human.var_names.str.startswith('MT-')  # annotate the group of mitochondrial genes as 'mt'\n",
    "sc.pp.calculate_qc_metrics(sole_2020_young_human, qc_vars=['mt'], percent_top=None, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.violin(sole_2020_young_human, ['n_genes_by_counts', 'total_counts', 'pct_counts_mt'],\n",
    "             jitter=0.4, multi_panel=True)\n",
    "\n",
    "sc.pl.scatter(sole_2020_young_human, x='total_counts', y='pct_counts_mt')\n",
    "sc.pl.scatter(sole_2020_young_human, x='total_counts', y='n_genes_by_counts')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1, 1, figsize=(10, 4))\n",
    "df = pd.DataFrame({'x': sole_2020_young_human.obs['Internal sample identifier'], 'y': sole_2020_young_human.obs['log1p_n_genes_by_counts']})\n",
    "sns.violinplot(x='x', y='y', data=df, ax=ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sole_2020_young_human = sole_2020_young_human[((sole_2020_young_human.obs.n_genes_by_counts < 2500) & \n",
    "                                    (sole_2020_young_human.obs.n_genes_by_counts > 350)).values, :]\n",
    "sole_2020_young_human = sole_2020_young_human[sole_2020_young_human.obs.pct_counts_mt < 15, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(sole_2020_young_human, min_counts=1)\n",
    "sc.pp.normalize_total(sole_2020_young_human)\n",
    "sc.pp.log1p(sole_2020_young_human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(sole_2020_young_human, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(sole_2020_young_human, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(sole_2020_young_human, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(sole_2020_young_human) ** 0.5 // 2), metric='cosine')\n",
    "tk.tl.triku(sole_2020_young_human, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(sole_2020_young_human, min_dist=0.6, random_state=seed)\n",
    "sc.tl.leiden(sole_2020_young_human, resolution=10, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(sole_2020_young_human, color=['Internal sample identifier', 'leiden', 'LUM', 'PDGFRA', 'COL1A1', 'DCN'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(sole_2020_young_human, color=['leiden', 'S100B', 'MPZ', 'PLP1', 'MLANA', 'PMEL'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(sole_2020_young_human, dict_cats=dict_cats_fb_human, min_score=0.3, quantile_gene_sel=0.8) # 0,2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(sole_2020_young_human, color=['Internal sample identifier', 'leiden', 'assigned_cats'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_fb_human.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(sole_2020_young_human, color=['assigned_cats'] + [i for i in val if i in sole_2020_young_human.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sole_2020_young_human_fb = sole_2020_young_human[sole_2020_young_human.obs['assigned_cats'].isin(['fibro', 'fibro - ANGPTL7'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(sole_2020_young_human_fb, min_counts=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(sole_2020_young_human_fb, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(sole_2020_young_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(sole_2020_young_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(sole_2020_young_human_fb) ** 0.5 // 2), metric='cosine')\n",
    "tk.tl.triku(sole_2020_young_human_fb, use_raw=False)\n",
    "\n",
    "sc.pp.pca(sole_2020_young_human_fb, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(sole_2020_young_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(sole_2020_young_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(sole_2020_young_human_fb) ** 0.5 // 2), metric='cosine')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(sole_2020_young_human_fb, min_dist=0.15, random_state=seed)\n",
    "sc.tl.leiden(sole_2020_young_human_fb, resolution=8, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(sole_2020_young_human_fb, dict_cats=dict_cats_clusters, min_score=0.5, quantile_gene_sel=0.9, key_added='cluster', others_name='U')\n",
    "assign_cats(sole_2020_young_human_fb, column_groupby='cluster', dict_cats=dict_cats_axes, min_score=0.5, quantile_gene_sel=0.9,  key_added='axis', \n",
    "            intermediate_states=True, diff=0.15, others_name='U')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sole_2020_young_human_fb.uns['cluster_colors'] = [dict_colors_human[i] if i in dict_colors_human else '#bcbcbc' for \n",
    "                                      i in sorted(set(sole_2020_young_human_fb.obs['cluster']))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(sole_2020_young_human_fb, color=['Internal sample identifier', 'leiden', 'axis', 'cluster'], legend_loc='on data', cmap=magma, use_raw=False, ncols=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot_score_graph(sole_2020_young_human_fb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "del sole_2020_young_human_fb.obs['C3']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_clusters.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(sole_2020_young_human_fb, color=['cluster'] + [i for i in val if i in sole_2020_young_human_fb.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "clear_adata(sole_2020_young_human_fb)\n",
    "sole_2020_young_human_fb.write_h5ad(sole_dir + '/sole_2020_young_human_fb_processed.h5')\n",
    "sole_2020_young_human.write_h5ad(sole_dir + '/sole_2020_young_human_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sole_2020_young_human_fb = sc.read(sole_dir + '/sole_2020_young_human_fb_processed.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tabib et al. 2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "tabib_2018_dir = data_dir + '/Tabib_2018'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "tabib_2018_ctrl_human = sc.read(tabib_2018_dir + '/adata_tabib_2018_ctrl_human.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_mettabib = pd.read_csv(tabib_2018_dir + '/Skin_6Control_Metadata.csv', index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "tabib_2018_ctrl_human"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "df metadata has 8366 cells, although the paper states that 8522 cells were analyzed. The rest of cells are erithrocytes, which were filtered out from the analysis."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "tabib_2018_ctrl_human.var_names = [dict_rep[i] if i in dict_rep else i for i in tabib_2018_ctrl_human.var_names]\n",
    "tabib_2018_ctrl_human.var['gene_symbol'] = tabib_2018_ctrl_human.var_names\n",
    "tabib_2018_ctrl_human.var['Ensemble ID'] = [dict_biomart_human[i] if i in dict_biomart_human else 'NA' for i in tabib_2018_ctrl_human.var_names]\n",
    "\n",
    "tabib_2018_ctrl_human.raw = tabib_2018_ctrl_human.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dict_reverse_mappings = {'Fibroblast': ['0', '3', '4'], \n",
    "                 'Keratinocyte': ['1', '5', '7', '11', '14',], \n",
    "                 'Endothelial cell': ['2'], \n",
    "                 'Pericyte': ['6', '10'], \n",
    "                 'Macrophage/DC': ['8'], \n",
    "                 'Lymphocyte': ['9'], \n",
    "                 'Secretory Epith': ['12'], \n",
    "                 'Smooth Muscle': ['13'], \n",
    "                 'Melanocyte': ['15'], \n",
    "                 'Neural Cell': ['16'],\n",
    "                 'Cornified Env': ['17'],\n",
    "                 'B cell': ['18'], \n",
    "                 'Erithrocyte': [np.NaN]}  # This is ours!\n",
    "\n",
    "dict_mappings = {}\n",
    "\n",
    "for key, val in dict_reverse_mappings.items():\n",
    "    for val_i in val:\n",
    "        dict_mappings[val_i] = key"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "tabib_2018_ctrl_human.obs['res.0.6'] = df_mettabib['res.0.6'].astype(str)\n",
    "tabib_2018_ctrl_human.obs['assigned_cats'] = [dict_mappings[i] for i in tabib_2018_ctrl_human.obs['res.0.6']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "tabib_2018_ctrl_human_fb = tabib_2018_ctrl_human[tabib_2018_ctrl_human.obs['assigned_cats'].isin(['Fibroblast']), :].copy()\n",
    "sc.pp.filter_genes(tabib_2018_ctrl_human_fb, min_counts=1)\n",
    "tabib_2018_ctrl_human_fb.X = spr.csr.csr_matrix(tabib_2018_ctrl_human_fb.X).copy()\n",
    "tabib_2018_ctrl_human_fb.raw = tabib_2018_ctrl_human_fb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.normalize_total(tabib_2018_ctrl_human_fb)\n",
    "sc.pp.log1p(tabib_2018_ctrl_human_fb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(tabib_2018_ctrl_human_fb, random_state=seed, n_comps=50)\n",
    "sce.pp.harmony_integrate(tabib_2018_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(tabib_2018_ctrl_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(len(tabib_2018_ctrl_human_fb) ** 0.5), metric='cosine')\n",
    "tk.tl.triku(tabib_2018_ctrl_human_fb, use_raw=False)\n",
    "\n",
    "sc.pp.pca(tabib_2018_ctrl_human_fb, random_state=seed, n_comps=50)\n",
    "sce.pp.harmony_integrate(tabib_2018_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(tabib_2018_ctrl_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(len(tabib_2018_ctrl_human_fb) ** 0.5), metric='cosine')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(tabib_2018_ctrl_human_fb, min_dist=0.35, random_state=seed)\n",
    "sc.tl.leiden(tabib_2018_ctrl_human_fb, resolution=16, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(tabib_2018_ctrl_human_fb, dict_cats=dict_cats_clusters, min_score=0.5, quantile_gene_sel=0.7, key_added='cluster', others_name='U')\n",
    "assign_cats(tabib_2018_ctrl_human_fb, column_groupby='cluster', dict_cats=dict_cats_axes, min_score=0.5, quantile_gene_sel=0.9, key_added='axis', \n",
    "            intermediate_states=True, diff=0.15, others_name='U')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "tabib_2018_ctrl_human_fb.uns['cluster_colors'] = [dict_colors_human[i] if i in dict_colors_human else '#bcbcbc' for \n",
    "                                      i in sorted(set(tabib_2018_ctrl_human_fb.obs['cluster']))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(tabib_2018_ctrl_human_fb, color=['Internal sample identifier', 'leiden', 'axis', 'cluster'], legend_loc='on data', \n",
    "           cmap=magma, use_raw=False, )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "del tabib_2018_ctrl_human_fb.obs['C3']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_clusters.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(tabib_2018_ctrl_human_fb, color=['cluster'] + [i for i in val if i in tabib_2018_ctrl_human_fb.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot_score_graph(tabib_2018_ctrl_human_fb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "clear_adata(tabib_2018_ctrl_human_fb)\n",
    "tabib_2018_ctrl_human_fb.write_h5ad(tabib_2018_dir + '/tabib_2018_ctrl_human_fb_processed.h5')\n",
    "tabib_2018_ctrl_human.write_h5ad(tabib_2018_dir + '/tabib_2018_ctrl_human_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "tabib_2018_ctrl_human_fb = sc.read(tabib_2018_dir + '/tabib_2018_ctrl_human_fb_processed.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Tabib et al. 2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "tabib_2021_dir = data_dir + '/Tabib_2021'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tabib_2021_ctrl_human = sc.read(tabib_2021_dir + '/adata_tabib_2021_ctrl_human.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tabib_2021_ctrl_human.var_names = [dict_rep[i] if i in dict_rep else i for i in tabib_2021_ctrl_human.var_names]\n",
    "tabib_2021_ctrl_human.var['gene_symbol'] = tabib_2021_ctrl_human.var_names\n",
    "tabib_2021_ctrl_human.var['Ensemble ID'] = [dict_biomart_human[i] if i in dict_biomart_human else 'NA' for i in tabib_2021_ctrl_human.var_names]\n",
    "\n",
    "tabib_2021_ctrl_human.raw = tabib_2021_ctrl_human.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Basic QC filtering\n",
    "tabib_2021_ctrl_human.var['mt'] = tabib_2021_ctrl_human.var_names.str.startswith('MT-')  # annotate the group of mitochondrial genes as 'mt'\n",
    "sc.pp.calculate_qc_metrics(tabib_2021_ctrl_human, qc_vars=['mt'], percent_top=None, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.violin(tabib_2021_ctrl_human, ['n_genes_by_counts', 'total_counts', 'pct_counts_mt'],\n",
    "             jitter=0.4, multi_panel=True)\n",
    "\n",
    "sc.pl.scatter(tabib_2021_ctrl_human, x='total_counts', y='pct_counts_mt')\n",
    "sc.pl.scatter(tabib_2021_ctrl_human, x='total_counts', y='n_genes_by_counts')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1, 1, figsize=(10, 4))\n",
    "df = pd.DataFrame({'x': tabib_2021_ctrl_human.obs['Internal sample identifier'], 'y': tabib_2021_ctrl_human.obs['log1p_n_genes_by_counts']})\n",
    "sns.violinplot(x='x', y='y', data=df, ax=ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1, 1, figsize=(10, 4))\n",
    "df = pd.DataFrame({'x': tabib_2021_ctrl_human.obs['Internal sample identifier'], 'y': tabib_2021_ctrl_human.obs['pct_counts_mt']})\n",
    "sns.violinplot(x='x', y='y', data=df, ax=ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "tabib_2021_ctrl_human = tabib_2021_ctrl_human[(((tabib_2021_ctrl_human.obs['Internal sample identifier'] == 'SC1') & (tabib_2021_ctrl_human.obs.log1p_n_genes_by_counts < 7.8) & \n",
    "                                                (tabib_2021_ctrl_human.obs.log1p_n_genes_by_counts > 6.5)) | \n",
    "                                              ((tabib_2021_ctrl_human.obs['Internal sample identifier'] == 'SC4') & (tabib_2021_ctrl_human.obs.log1p_n_genes_by_counts < 7.8) & \n",
    "                                                (tabib_2021_ctrl_human.obs.log1p_n_genes_by_counts > 6.5)) | \n",
    "                                              ((tabib_2021_ctrl_human.obs['Internal sample identifier'] == 'SC18') & (tabib_2021_ctrl_human.obs.log1p_n_genes_by_counts < 7.8) & \n",
    "                                                (tabib_2021_ctrl_human.obs.log1p_n_genes_by_counts > 6.5)) | \n",
    "                                              ((tabib_2021_ctrl_human.obs['Internal sample identifier'] == 'SC32') & (tabib_2021_ctrl_human.obs.log1p_n_genes_by_counts < 7.8) & \n",
    "                                                (tabib_2021_ctrl_human.obs.log1p_n_genes_by_counts > 6.5)) | \n",
    "                                              ((tabib_2021_ctrl_human.obs['Internal sample identifier'] == 'SC33') & (tabib_2021_ctrl_human.obs.log1p_n_genes_by_counts < 7.8) & \n",
    "                                                (tabib_2021_ctrl_human.obs.log1p_n_genes_by_counts > 6.5)) | \n",
    "                                              ((tabib_2021_ctrl_human.obs['Internal sample identifier'] == 'SC34') & (tabib_2021_ctrl_human.obs.log1p_n_genes_by_counts < 7.8) & \n",
    "                                                (tabib_2021_ctrl_human.obs.log1p_n_genes_by_counts > 6.5)) | \n",
    "                                              ((tabib_2021_ctrl_human.obs['Internal sample identifier'] == 'SC50') & (tabib_2021_ctrl_human.obs.log1p_n_genes_by_counts < 7.8) & \n",
    "                                                (tabib_2021_ctrl_human.obs.log1p_n_genes_by_counts > 6.5)) | \n",
    "                                              ((tabib_2021_ctrl_human.obs['Internal sample identifier'] == 'SC68') & (tabib_2021_ctrl_human.obs.log1p_n_genes_by_counts < 7.8) & \n",
    "                                                (tabib_2021_ctrl_human.obs.log1p_n_genes_by_counts > 6.5)) | \n",
    "                                              ((tabib_2021_ctrl_human.obs['Internal sample identifier'] == 'SC124') & (tabib_2021_ctrl_human.obs.log1p_n_genes_by_counts < 7.8) & \n",
    "                                                (tabib_2021_ctrl_human.obs.log1p_n_genes_by_counts > 6.5)) | \n",
    "                                              ((tabib_2021_ctrl_human.obs['Internal sample identifier'] == 'SC125') & (tabib_2021_ctrl_human.obs.log1p_n_genes_by_counts < 7.8) & \n",
    "                                                (tabib_2021_ctrl_human.obs.log1p_n_genes_by_counts > 6.5))).values, :]\n",
    "tabib_2021_ctrl_human = tabib_2021_ctrl_human[tabib_2021_ctrl_human.obs.pct_counts_mt < 15, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(tabib_2021_ctrl_human, min_counts=1)\n",
    "sc.pp.normalize_total(tabib_2021_ctrl_human)\n",
    "sc.pp.log1p(tabib_2021_ctrl_human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc.pp.pca(tabib_2021_ctrl_human, random_state=seed, n_comps=50)\n",
    "sce.pp.harmony_integrate(tabib_2021_ctrl_human, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(tabib_2021_ctrl_human, use_rep='X_pca_harmony', n_neighbors=int(0.5 * len(tabib_2021_ctrl_human) ** 0.5 // 2), metric='cosine')\n",
    "tk.tl.triku(tabib_2021_ctrl_human, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc.tl.umap(tabib_2021_ctrl_human, min_dist=0.2, random_state=seed)\n",
    "sc.tl.leiden(tabib_2021_ctrl_human, resolution=1.3, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc.pp.subsample(tabib_2021_ctrl_human, fraction=1, random_state=0, copy=False)\n",
    "sc.pl.umap(tabib_2021_ctrl_human, color=['leiden', 'Internal sample identifier'], legend_loc='on data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(tabib_2021_ctrl_human, color=['leiden', 'LUM', 'PDGFRA', 'COL1A1', 'DCN', 'RGS5', 'KRT5', 'DMKN'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc.pl.umap(tabib_2021_ctrl_human, color=['leiden', 'C2orf40', 'CDH19', 'ANGPTL7','PLEKHB1','ENTPD2', \n",
    "                                   'SLC2A1', 'CLDN1', 'TNNT2', 'C19orf33', 'SFRP5', 'WNT6', ], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assign_cats(tabib_2021_ctrl_human, dict_cats=dict_cats_fb_human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_fb_human.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(tabib_2021_ctrl_human, color=['assigned_cats'] + [i for i in val if i in tabib_2021_ctrl_human.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(tabib_2021_ctrl_human, color=['leiden', 'assigned_cats'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "tabib_2021_ctrl_human_fb = tabib_2021_ctrl_human[tabib_2021_ctrl_human.obs['assigned_cats'].isin(['fibro', 'fibro - ANGPTL7'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(tabib_2021_ctrl_human_fb, min_counts=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(tabib_2021_ctrl_human_fb, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(tabib_2021_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(tabib_2021_ctrl_human_fb, use_rep='X_pca_harmony', n_neighbors=int(0.5 * len(tabib_2021_ctrl_human_fb) ** 0.5 // 5), metric='cosine')\n",
    "tk.tl.triku(tabib_2021_ctrl_human_fb, use_raw=False)\n",
    "\n",
    "sc.pp.pca(tabib_2021_ctrl_human_fb, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(tabib_2021_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(tabib_2021_ctrl_human_fb, use_rep='X_pca_harmony', n_neighbors=int(0.5 * len(tabib_2021_ctrl_human_fb) ** 0.5 // 5), metric='cosine')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(tabib_2021_ctrl_human_fb, min_dist=0.25, random_state=seed)\n",
    "sc.tl.leiden(tabib_2021_ctrl_human_fb, resolution=6, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(tabib_2021_ctrl_human_fb, dict_cats=dict_cats_clusters, min_score=0.4, quantile_gene_sel=0.7, key_added='cluster', others_name='U')\n",
    "assign_cats(tabib_2021_ctrl_human_fb, column_groupby='cluster', dict_cats=dict_cats_axes, min_score=0.5, quantile_gene_sel=0.9,  key_added='axis', \n",
    "             intermediate_states=True, diff=0.15, others_name='U')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "tabib_2021_ctrl_human_fb.uns['cluster_colors'] = [dict_colors_human[i] if i in dict_colors_human else '#bcbcbc' for \n",
    "                                      i in sorted(set(tabib_2021_ctrl_human_fb.obs['cluster']))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(tabib_2021_ctrl_human_fb, color=['Internal sample identifier', 'leiden', 'axis', 'cluster'], legend_loc='on data', cmap=magma, use_raw=False, ncols=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(tabib_2021_ctrl_human_fb, color=['Internal sample identifier', 'leiden', 'axis', 'cluster'], legend_loc='on data', cmap=magma, use_raw=False, ncols=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "del tabib_2021_ctrl_human_fb.obs['C3']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_clusters.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(tabib_2021_ctrl_human_fb, color=['cluster'] + [i for i in val if i in tabib_2021_ctrl_human_fb.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot_score_graph(tabib_2021_ctrl_human_fb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "clear_adata(tabib_2021_ctrl_human_fb)\n",
    "tabib_2021_ctrl_human_fb.write_h5ad(tabib_2021_dir + '/tabib_2021_ctrl_human_fb_processed.h5')\n",
    "tabib_2021_ctrl_human.write_h5ad(tabib_2021_dir + '/tabib_2021_ctrl_human_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "tabib_2021_ctrl_human_fb = sc.read(tabib_2021_dir + '/tabib_2021_ctrl_human_fb_processed.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Tabula Sapiens Consortium 2021 [Not included because they do not yield good quality populations]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tsc_dir = data_dir + '/Tabula_Sapiens_Consortium_2021'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tsc_2021_ctrl_human = sc.read(tsc_dir + '/adata_tsc_2021_ctrl_human.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(tsc_2021_ctrl_human, min_counts=25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tsc_2021_ctrl_human.var_names = [dict_rep[i] if i in dict_rep else i for i in tsc_2021_ctrl_human.var_names]\n",
    "tsc_2021_ctrl_human.var['gene_symbol'] = tsc_2021_ctrl_human.var_names\n",
    "tsc_2021_ctrl_human.var['Ensemble ID'] = [dict_biomart_human[i] if i in dict_biomart_human else 'NA' for i in tsc_2021_ctrl_human.var_names]\n",
    "\n",
    "tsc_2021_ctrl_human.raw = tsc_2021_ctrl_human.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Basic QC filtering\n",
    "tsc_2021_ctrl_human.var['mt'] = tsc_2021_ctrl_human.var_names.str.startswith('MT-')  # annotate the group of mitochondrial genes as 'mt'\n",
    "sc.pp.calculate_qc_metrics(tsc_2021_ctrl_human, qc_vars=['mt'], percent_top=None, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc.pl.violin(tsc_2021_ctrl_human, ['n_genes_by_counts', 'total_counts', 'pct_counts_mt'],\n",
    "             jitter=0.4, multi_panel=True)\n",
    "\n",
    "sc.pl.scatter(tsc_2021_ctrl_human, x='total_counts', y='pct_counts_mt')\n",
    "sc.pl.scatter(tsc_2021_ctrl_human, x='total_counts', y='n_genes_by_counts')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1, 1, figsize=(10, 4))\n",
    "df = pd.DataFrame({'x': tsc_2021_ctrl_human.obs['Internal sample identifier'], 'y': tsc_2021_ctrl_human.obs['log1p_n_genes_by_counts']})\n",
    "sns.violinplot(x='x', y='y', data=df, ax=ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1, 1, figsize=(10, 4))\n",
    "df = pd.DataFrame({'x': tsc_2021_ctrl_human.obs['Internal sample identifier'], 'y': tsc_2021_ctrl_human.obs['pct_counts_mt']})\n",
    "sns.violinplot(x='x', y='y', data=df, ax=ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "tsc_2021_ctrl_human = tsc_2021_ctrl_human[(((tsc_2021_ctrl_human.obs['Internal sample identifier'] == 'T10_S5') & (tsc_2021_ctrl_human.obs.log1p_n_genes_by_counts < 8) & \n",
    "                        (tsc_2021_ctrl_human.obs.log1p_n_genes_by_counts > 6.8)) | \n",
    "                      ((tsc_2021_ctrl_human.obs['Internal sample identifier'] == 'T10_S6') & (tsc_2021_ctrl_human.obs.log1p_n_genes_by_counts < 8) & \n",
    "                        (tsc_2021_ctrl_human.obs.log1p_n_genes_by_counts > 6.8)) | \n",
    "                      ((tsc_2021_ctrl_human.obs['Internal sample identifier'] == 'T14_S17') & (tsc_2021_ctrl_human.obs.log1p_n_genes_by_counts < 8.1) & \n",
    "                        (tsc_2021_ctrl_human.obs.log1p_n_genes_by_counts > 6.8)) | \n",
    "                      ((tsc_2021_ctrl_human.obs['Internal sample identifier'] == 'T14_S18') & (tsc_2021_ctrl_human.obs.log1p_n_genes_by_counts < 8.1) & \n",
    "                        (tsc_2021_ctrl_human.obs.log1p_n_genes_by_counts > 6.8)) ).values, :]\n",
    "tsc_2021_ctrl_human = tsc_2021_ctrl_human[tsc_2021_ctrl_human.obs.pct_counts_mt < 15, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(tsc_2021_ctrl_human, min_counts=1)\n",
    "sc.pp.normalize_total(tsc_2021_ctrl_human)\n",
    "sc.pp.log1p(tsc_2021_ctrl_human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(tsc_2021_ctrl_human, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(tsc_2021_ctrl_human, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(tsc_2021_ctrl_human, use_rep='X_pca_harmony', n_neighbors=int(0.5 * len(tsc_2021_ctrl_human) ** 0.5), metric='cosine')\n",
    "tk.tl.triku(tsc_2021_ctrl_human, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc.tl.umap(tsc_2021_ctrl_human, min_dist=0.3, random_state=seed)\n",
    "sc.tl.leiden(tsc_2021_ctrl_human, resolution=1.8, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(tsc_2021_ctrl_human, color=['leiden', 'LUM', 'PDGFRA', 'COL1A1', 'DCN', 'RGS5'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(tsc_2021_ctrl_human, dict_cats=dict_cats_fb_human, min_score=0.3, quantile_gene_sel=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(tsc_2021_ctrl_human, color=['Internal sample identifier', 'leiden', 'assigned_cats'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tsc_2021_ctrl_human_fb = tsc_2021_ctrl_human[tsc_2021_ctrl_human.obs['assigned_cats'] == 'fibro']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(tsc_2021_ctrl_human_fb, min_counts=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(tsc_2021_ctrl_human_fb, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(tsc_2021_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(tsc_2021_ctrl_human_fb, use_rep='X_pca_harmony', n_neighbors=int(0.3 * len(tsc_2021_ctrl_human_fb) ** 0.5), metric='cosine')\n",
    "tk.tl.triku(tsc_2021_ctrl_human_fb, use_raw=False)\n",
    "\n",
    "sc.pp.pca(tsc_2021_ctrl_human_fb, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(tsc_2021_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(tsc_2021_ctrl_human_fb, use_rep='X_pca_harmony', n_neighbors=int(0.3 * len(tsc_2021_ctrl_human_fb) ** 0.5), metric='cosine')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc.tl.umap(tsc_2021_ctrl_human_fb, min_dist=0.25, random_state=seed)\n",
    "sc.tl.leiden(tsc_2021_ctrl_human_fb, resolution=12, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assign_cats(tsc_2021_ctrl_human_fb, dict_cats=dict_cats_clusters, min_score=0.6, quantile_gene_sel=0.99, key_added='cluster', others_name='U')\n",
    "assign_cats(tsc_2021_ctrl_human_fb, column_groupby='cluster', dict_cats=dict_cats_axes, min_score=0.4, key_added='axis', \n",
    "            intermediate_states=True, diff=0.15, others_name='U')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "tsc_2021_ctrl_human_fb.uns['cluster_colors'] = [dict_colors_human[i] if i in dict_colors_human else '#bcbcbc' for \n",
    "                                      i in sorted(set(tsc_2021_ctrl_human_fb.obs['cluster']))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(tsc_2021_ctrl_human_fb, color=['Internal sample identifier', 'leiden', 'axis', 'cluster'], legend_loc='on data', \n",
    "           cmap=magma, use_raw=False, )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "del tsc_2021_ctrl_human_fb.obs['C3']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_clusters.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(tsc_2021_ctrl_human_fb, color=['cluster'] + [i for i in val if i in tsc_2021_ctrl_human_fb.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot_score_graph(tsc_2021_ctrl_human_fb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "clear_adata(tsc_2021_ctrl_human_fb)\n",
    "tsc_2021_ctrl_human_fb.write_h5ad(tsc_dir + '/tsc_2021_ctrl_human_fb_processed.h5')\n",
    "tsc_2021_ctrl_human.write_h5ad(tsc_dir + '/tsc_2021_ctrl_human_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "tsc_2021_ctrl_human_fb = sc.read(tsc_dir + '/tsc_2021_ctrl_human_fb_processed.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Theocarditis 2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "theo_dir_2020 = data_dir + '/Theocharidis_2020/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "theo_2020_ctrl_human = sc.read(theo_dir_2020 + '/adata_theo_ctrl_human.h5')\n",
    "theo_2020_dm = sc.read(theo_dir_2020 + '/adata_theo_dm_noDFU.h5')\n",
    "\n",
    "theo_2020_ctrl_human_dm = sc.AnnData.concatenate(theo_2020_ctrl_human, theo_2020_dm, batch_key='condition', batch_categories=['healthy', 'DM'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(theo_2020_ctrl_human_dm, min_counts=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "theo_2020_ctrl_human_dm.var_names = [dict_rep[i] if i in dict_rep else i for i in theo_2020_ctrl_human_dm.var_names]\n",
    "theo_2020_ctrl_human_dm.var['gene_symbol'] = theo_2020_ctrl_human_dm.var_names\n",
    "theo_2020_ctrl_human_dm.var['Ensemble ID'] = [dict_biomart_human[i] if i in dict_biomart_human else 'NA' for i in theo_2020_ctrl_human_dm.var_names]\n",
    "\n",
    "theo_2020_ctrl_human_dm.raw = theo_2020_ctrl_human_dm.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Basic QC filtering\n",
    "theo_2020_ctrl_human_dm.var['mt'] = theo_2020_ctrl_human_dm.var_names.str.startswith('MT-')  # annotate the group of mitochondrial genes as 'mt'\n",
    "sc.pp.calculate_qc_metrics(theo_2020_ctrl_human_dm, qc_vars=['mt'], percent_top=None, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "theo_2020_ctrl_human_dm.X = np.array(theo_2020_ctrl_human_dm.X.todense())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc.pl.violin(theo_2020_ctrl_human_dm, ['n_genes_by_counts', 'total_counts', 'pct_counts_mt'],\n",
    "             jitter=0.4, multi_panel=True)\n",
    "\n",
    "sc.pl.scatter(theo_2020_ctrl_human_dm, x='total_counts', y='pct_counts_mt')\n",
    "sc.pl.scatter(theo_2020_ctrl_human_dm, x='total_counts', y='n_genes_by_counts')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1, 1, figsize=(10, 4))\n",
    "df = pd.DataFrame({'x': theo_2020_ctrl_human_dm.obs['Internal sample identifier'], 'y': theo_2020_ctrl_human_dm.obs['log1p_n_genes_by_counts']})\n",
    "sns.violinplot(x='x', y='y', data=df, ax=ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1, 1, figsize=(10, 4))\n",
    "df = pd.DataFrame({'x': theo_2020_ctrl_human_dm.obs['Internal sample identifier'], 'y': theo_2020_ctrl_human_dm.obs['pct_counts_mt']})\n",
    "sns.violinplot(x='x', y='y', data=df, ax=ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "theo_2020_ctrl_human_dm = theo_2020_ctrl_human_dm[(((theo_2020_ctrl_human_dm.obs['Internal sample identifier'] == 'H1') & (theo_2020_ctrl_human_dm.obs.log1p_n_genes_by_counts < 7.6) & \n",
    "                        (theo_2020_ctrl_human_dm.obs.log1p_n_genes_by_counts > 6.2)) | \n",
    "                      ((theo_2020_ctrl_human_dm.obs['Internal sample identifier'] == 'H2') & (theo_2020_ctrl_human_dm.obs.log1p_n_genes_by_counts < 7.6) & \n",
    "                        (theo_2020_ctrl_human_dm.obs.log1p_n_genes_by_counts > 6.3)) | \n",
    "                      ((theo_2020_ctrl_human_dm.obs['Internal sample identifier'] == 'H3') & (theo_2020_ctrl_human_dm.obs.log1p_n_genes_by_counts < 7.6) & \n",
    "                        (theo_2020_ctrl_human_dm.obs.log1p_n_genes_by_counts > 6.4)) | \n",
    "                      ((theo_2020_ctrl_human_dm.obs['Internal sample identifier'] == 'H4') & (theo_2020_ctrl_human_dm.obs.log1p_n_genes_by_counts < 7.6) & \n",
    "                        (theo_2020_ctrl_human_dm.obs.log1p_n_genes_by_counts > 6.4)) |\n",
    "                      ((theo_2020_ctrl_human_dm.obs['Internal sample identifier'] == 'DM2') & (theo_2020_ctrl_human_dm.obs.log1p_n_genes_by_counts < 8) & \n",
    "                        (theo_2020_ctrl_human_dm.obs.log1p_n_genes_by_counts > 6.7)) | \n",
    "                      ((theo_2020_ctrl_human_dm.obs['Internal sample identifier'] == 'DM3') & (theo_2020_ctrl_human_dm.obs.log1p_n_genes_by_counts < 7.4) & \n",
    "                        (theo_2020_ctrl_human_dm.obs.log1p_n_genes_by_counts > 6.5)) | \n",
    "                                         ((theo_2020_ctrl_human_dm.obs['Internal sample identifier'] == 'DM4') & (theo_2020_ctrl_human_dm.obs.log1p_n_genes_by_counts < 7.6) & \n",
    "                        (theo_2020_ctrl_human_dm.obs.log1p_n_genes_by_counts > 6.5))  ).values, :]\n",
    "theo_2020_ctrl_human_dm = theo_2020_ctrl_human_dm[theo_2020_ctrl_human_dm.obs.pct_counts_mt < 12, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Basic QC filtering\n",
    "theo_2020_ctrl_human_dm.var['mt'] = theo_2020_ctrl_human_dm.var_names.str.startswith('MT-')  # annotate the group of mitochondrial genes as 'mt'\n",
    "sc.pp.calculate_qc_metrics(theo_2020_ctrl_human_dm, qc_vars=['mt'], percent_top=None, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(theo_2020_ctrl_human_dm, min_counts=1)\n",
    "sc.pp.normalize_total(theo_2020_ctrl_human_dm)\n",
    "sc.pp.log1p(theo_2020_ctrl_human_dm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(theo_2020_ctrl_human_dm, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(theo_2020_ctrl_human_dm, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(theo_2020_ctrl_human_dm, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(theo_2020_ctrl_human_dm) ** 0.5 // 4), metric='cosine')\n",
    "tk.tl.triku(theo_2020_ctrl_human_dm, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc.tl.umap(theo_2020_ctrl_human_dm, min_dist=0.3, random_state=seed)\n",
    "sc.tl.leiden(theo_2020_ctrl_human_dm, resolution=3, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(theo_2020_ctrl_human_dm, color=['leiden', 'LUM', 'PDGFRA', 'COL1A1', 'DCN', 'RGS5'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(theo_2020_ctrl_human_dm, dict_cats=dict_cats_fb_human, min_score=0.4, quantile_gene_sel=0.97)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_fb_human.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(theo_2020_ctrl_human_dm, color=['assigned_cats'] + [i for i in val if i in theo_2020_ctrl_human_dm.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(theo_2020_ctrl_human_dm, color=['leiden', 'Internal sample identifier', 'condition', 'assigned_cats'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(theo_2020_ctrl_human_dm, color=['leiden', 'COL18A1', 'APCDD1', 'SLPI', 'CCL19', 'GGT5',  'CHRDL1', 'GPX3', 'BGN', 'ASPN', 'TNN', 'COL11A1', 'COCH', \n",
    "                                     'IGFBP3', 'ANGPTL7', 'SCN7A', 'C2orf40', 'NGFR', 'CLDN1', 'SBSPON', 'FGFBP2', 'DIO3', 'LUZP2'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "theo_2020_ctrl_human_dm_fb = theo_2020_ctrl_human_dm[theo_2020_ctrl_human_dm.obs['assigned_cats'].isin(['fibro', 'fibro - ANGPTL7', 'F'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(theo_2020_ctrl_human_dm_fb, min_counts=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(theo_2020_ctrl_human_dm_fb, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(theo_2020_ctrl_human_dm_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(theo_2020_ctrl_human_dm_fb, use_rep='X_pca_harmony',  n_neighbors=int(len(theo_2020_ctrl_human_dm_fb) ** 0.5 // 4), metric='cosine')\n",
    "tk.tl.triku(theo_2020_ctrl_human_dm_fb, use_raw=False)\n",
    "\n",
    "sc.pp.pca(theo_2020_ctrl_human_dm_fb, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(theo_2020_ctrl_human_dm_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(theo_2020_ctrl_human_dm_fb, use_rep='X_pca_harmony',  n_neighbors=int(len(theo_2020_ctrl_human_dm_fb) ** 0.5 // 4), metric='cosine')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(theo_2020_ctrl_human_dm_fb, min_dist=0.15, random_state=seed)\n",
    "sc.tl.leiden(theo_2020_ctrl_human_dm_fb, resolution=7, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(theo_2020_ctrl_human_dm_fb, dict_cats=dict_cats_clusters, min_score=0.5, quantile_gene_sel=0.8, key_added='cluster', others_name='U')\n",
    "assign_cats(theo_2020_ctrl_human_dm_fb, column_groupby='cluster', dict_cats=dict_cats_axes, min_score=0.4, key_added='axis', \n",
    "            intermediate_states=True, diff=0.15, others_name='U')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "theo_2020_ctrl_human_dm_fb.uns['cluster_colors'] = [dict_colors_human[i] if i in dict_colors_human else '#bcbcbc' for \n",
    "                                      i in sorted(set(theo_2020_ctrl_human_dm_fb.obs['cluster']))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(theo_2020_ctrl_human_dm_fb, color=['Internal sample identifier', 'leiden', 'axis',  'cluster', 'condition'], legend_loc='on data', \n",
    "           cmap=magma, use_raw=False, )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "del theo_2020_ctrl_human_dm_fb.obs['C3']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_clusters.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(theo_2020_ctrl_human_dm_fb, color=['cluster'] + [i for i in val if i in theo_2020_ctrl_human_dm_fb.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot_score_graph(theo_2020_ctrl_human_dm_fb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "clear_adata(theo_2020_ctrl_human_dm_fb)\n",
    "theo_2020_ctrl_human_dm_fb.write_h5ad(theo_dir_2020 + '/theo_2020_ctrl_human_dm_fb_processed.h5')\n",
    "theo_2020_ctrl_human_dm.write_h5ad(theo_dir_2020 + '/theo_2020_ctrl_human_dm_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "theo_2020_ctrl_human_dm_fb = sc.read(theo_dir_2020 + '/theo_2020_ctrl_human_dm_fb_processed.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Theocarditis 2021\n",
    "\n",
    "The dataset presents two B1 populations (CXCL8, CA12, COL7A1, CD82, TMEM158, WNT5A, C15orf48, SAT1, MT2A, IER3, PTGS2, SRGN, CXCL1, CXCL3, CCL3, HMGA1, STC1, UPP1, ATP13A3, POU2F2, BCL2A1, F3, S1PR3, PMAIP1, LAMB3, IL1B, MMP9, SLC7A5, CDCP1, EGLN3, GMFG) and (IGFBP4, A2M, IGF1, FGF7, MEDAG, CHI3L1, SFRP1, IGFBP2, LXN, GPC3, PALMD, ALPL, CXCL2, CCDC69). These two populations could not be replicated in the rest of datasets, so we do not consider them as two distinct populations, but rather as a possible artifact."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "theo_dir_2021 = data_dir + '/Theocharidis_2021/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "theo_2021_all = sc.read(theo_dir_2021 + '/adata_theo_2021.h5')\n",
    "theo_2021_ctrl_human = theo_2021_all[theo_2021_all.obs['Condition'] == 'Healthy']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(theo_2021_ctrl_human, min_counts=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "theo_2021_ctrl_human.var_names = [dict_rep[i] if i in dict_rep else i for i in theo_2021_ctrl_human.var_names]\n",
    "theo_2021_ctrl_human.var['gene_symbol'] = theo_2021_ctrl_human.var_names\n",
    "theo_2021_ctrl_human.var['Ensemble ID'] = [dict_biomart_human[i] if i in dict_biomart_human else 'NA' for i in theo_2021_ctrl_human.var_names]\n",
    "\n",
    "theo_2021_ctrl_human.raw = theo_2021_ctrl_human.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "theo_2021_ctrl_human"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Basic QC filtering\n",
    "theo_2021_ctrl_human.var['mt'] = theo_2021_ctrl_human.var_names.str.startswith('MT-')  # annotate the group of mitochondrial genes as 'mt'\n",
    "sc.pp.calculate_qc_metrics(theo_2021_ctrl_human, qc_vars=['mt'], percent_top=None, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.violin(theo_2021_ctrl_human, ['n_genes_by_counts', 'total_counts', 'pct_counts_mt'],\n",
    "             jitter=0.4, multi_panel=True)\n",
    "\n",
    "sc.pl.scatter(theo_2021_ctrl_human, x='total_counts', y='pct_counts_mt')\n",
    "sc.pl.scatter(theo_2021_ctrl_human, x='total_counts', y='n_genes_by_counts')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1, 1, figsize=(10, 4))\n",
    "df = pd.DataFrame({'x': theo_2021_ctrl_human.obs['Internal sample identifier'], 'y': theo_2021_ctrl_human.obs['log1p_n_genes_by_counts']})\n",
    "sns.violinplot(x='x', y='y', data=df, ax=ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1, 1, figsize=(10, 4))\n",
    "df = pd.DataFrame({'x': theo_2021_ctrl_human.obs['Internal sample identifier'], 'y': theo_2021_ctrl_human.obs['pct_counts_mt']})\n",
    "sns.violinplot(x='x', y='y', data=df, ax=ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "theo_2021_ctrl_human = theo_2021_ctrl_human[((theo_2021_ctrl_human.obs.n_genes_by_counts < 2750) & \n",
    "                                    (theo_2021_ctrl_human.obs.n_genes_by_counts > 900)).values, :]\n",
    "theo_2021_ctrl_human = theo_2021_ctrl_human[theo_2021_ctrl_human.obs.pct_counts_mt < 15, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Basic QC filtering\n",
    "theo_2021_ctrl_human.var['mt'] = theo_2021_ctrl_human.var_names.str.startswith('MT-')  # annotate the group of mitochondrial genes as 'mt'\n",
    "sc.pp.calculate_qc_metrics(theo_2021_ctrl_human, qc_vars=['mt'], percent_top=None, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(theo_2021_ctrl_human, min_counts=1)\n",
    "sc.pp.normalize_total(theo_2021_ctrl_human)\n",
    "sc.pp.log1p(theo_2021_ctrl_human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(theo_2021_ctrl_human, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(theo_2021_ctrl_human, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(theo_2021_ctrl_human, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(theo_2021_ctrl_human) ** 0.5 // 4), metric='cosine')\n",
    "tk.tl.triku(theo_2021_ctrl_human, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(theo_2021_ctrl_human, min_dist=0.3, random_state=seed)\n",
    "sc.tl.leiden(theo_2021_ctrl_human, resolution=3, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(theo_2021_ctrl_human, color=['leiden', 'LUM', 'PDGFRA', 'COL1A1', 'DCN', 'RGS5'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(theo_2021_ctrl_human, dict_cats=dict_cats_fb_human, min_score=0.4, quantile_gene_sel=0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_fb_human.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(theo_2021_ctrl_human, color=['assigned_cats'] + [i for i in val if i in theo_2021_ctrl_human.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(theo_2021_ctrl_human, color=['Internal sample identifier', 'leiden', 'Condition', 'assigned_cats'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "theo_2021_ctrl_human_fb = theo_2021_ctrl_human[theo_2021_ctrl_human.obs['assigned_cats'].isin(['fibro'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(theo_2021_ctrl_human_fb, min_counts=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(theo_2021_ctrl_human_fb, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(theo_2021_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(theo_2021_ctrl_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(len(theo_2021_ctrl_human_fb) ** 0.5 // 4), metric='cosine')\n",
    "tk.tl.triku(theo_2021_ctrl_human_fb, use_raw=False)\n",
    "\n",
    "sc.pp.pca(theo_2021_ctrl_human_fb, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(theo_2021_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(theo_2021_ctrl_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(len(theo_2021_ctrl_human_fb) ** 0.5 // 4), metric='cosine')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(theo_2021_ctrl_human_fb, min_dist=0.15, random_state=seed)\n",
    "sc.tl.leiden(theo_2021_ctrl_human_fb, resolution=4, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(theo_2021_ctrl_human_fb, dict_cats=dict_cats_clusters, min_score=0.5, quantile_gene_sel=0.65, key_added='cluster', others_name='U')\n",
    "assign_cats(theo_2021_ctrl_human_fb, column_groupby='cluster', dict_cats=dict_cats_axes, min_score=0.4, key_added='axis', \n",
    "            intermediate_states=True, diff=0.15, others_name='U')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "theo_2021_ctrl_human_fb.uns['cluster_colors'] = [dict_colors_human[i] if i in dict_colors_human else '#bcbcbc' for \n",
    "                                      i in sorted(set(theo_2021_ctrl_human_fb.obs['cluster']))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(theo_2021_ctrl_human_fb, color=['Internal sample identifier', 'leiden', 'axis', 'cluster'], legend_loc='on data', \n",
    "           cmap=magma, use_raw=False, )\n",
    "sc.pl.umap(theo_2021_ctrl_human_fb, color=['Condition', 'Sample location'], cmap=magma, use_raw=False, )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "del theo_2021_ctrl_human_fb.obs['C3']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_clusters.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(theo_2021_ctrl_human_fb, color=['cluster'] + [i for i in val if i in theo_2021_ctrl_human_fb.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot_score_graph(theo_2021_ctrl_human_fb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "clear_adata(theo_2021_ctrl_human_fb)\n",
    "theo_2021_ctrl_human_fb.write_h5ad(theo_dir_2021 + '/theo_2021_ctrl_human_fb_processed.h5')\n",
    "theo_2021_ctrl_human.write_h5ad(theo_dir_2021 + '/theo_2021_ctrl_human_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "theo_2021_ctrl_human_fb = sc.read(theo_dir_2021 + '/theo_2021_ctrl_human_fb_processed.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Vorstandlechner et al. 2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "vors_dir = data_dir + '/Vorstandlechner_2020'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "vors_2020_ctrl_human = sc.read(vors_dir + '/adata_vors_2020_ctrl_human.h5', cache=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vors_2020_ctrl_human.var_names = [dict_rep[i] if i in dict_rep else i for i in vors_2020_ctrl_human.var_names]\n",
    "vors_2020_ctrl_human.var['gene_symbol'] = vors_2020_ctrl_human.var_names\n",
    "vors_2020_ctrl_human.var['Ensemble ID'] = [dict_biomart_human[i] if i in dict_biomart_human else 'NA' for i in vors_2020_ctrl_human.var_names]\n",
    "\n",
    "vors_2020_ctrl_human.raw = vors_2020_ctrl_human.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(vors_2020_ctrl_human, min_counts=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Basic QC filtering\n",
    "vors_2020_ctrl_human.var['mt'] = vors_2020_ctrl_human.var_names.str.startswith('MT-')  # annotate the group of mitochondrial genes as 'mt'\n",
    "sc.pp.calculate_qc_metrics(vors_2020_ctrl_human, qc_vars=['mt'], percent_top=None, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc.pl.violin(vors_2020_ctrl_human, ['n_genes_by_counts', 'total_counts', 'pct_counts_mt'],\n",
    "             jitter=0.4, multi_panel=True)\n",
    "\n",
    "sc.pl.scatter(vors_2020_ctrl_human, x='total_counts', y='pct_counts_mt')\n",
    "sc.pl.scatter(vors_2020_ctrl_human, x='total_counts', y='n_genes_by_counts')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1, 1, figsize=(10, 4))\n",
    "df = pd.DataFrame({'x': vors_2020_ctrl_human.obs['Internal sample identifier'], 'y': vors_2020_ctrl_human.obs['log1p_n_genes_by_counts']})\n",
    "sns.violinplot(x='x', y='y', data=df, ax=ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1, 1, figsize=(10, 4))\n",
    "df = pd.DataFrame({'x': vors_2020_ctrl_human.obs['Internal sample identifier'], 'y': vors_2020_ctrl_human.obs['pct_counts_mt']})\n",
    "sns.violinplot(x='x', y='y', data=df, ax=ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vors_2020_ctrl_human = vors_2020_ctrl_human[(((vors_2020_ctrl_human.obs['Internal sample identifier'] == '1') & (vors_2020_ctrl_human.obs.log1p_n_genes_by_counts < 6.2) & \n",
    "                        (vors_2020_ctrl_human.obs.log1p_n_genes_by_counts > 5.6)) | \n",
    "                      ((vors_2020_ctrl_human.obs['Internal sample identifier'] == '2') & (vors_2020_ctrl_human.obs.log1p_n_genes_by_counts < 7.4) & \n",
    "                        (vors_2020_ctrl_human.obs.log1p_n_genes_by_counts > 6)) | \n",
    "                      ((vors_2020_ctrl_human.obs['Internal sample identifier'] == '3') & (vors_2020_ctrl_human.obs.log1p_n_genes_by_counts < 7.4) & \n",
    "                        (vors_2020_ctrl_human.obs.log1p_n_genes_by_counts > 5.8))).values, :]\n",
    "vors_2020_ctrl_human = vors_2020_ctrl_human[vors_2020_ctrl_human.obs.pct_counts_mt < 10, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(vors_2020_ctrl_human, min_counts=1)\n",
    "sc.pp.normalize_total(vors_2020_ctrl_human)\n",
    "sc.pp.log1p(vors_2020_ctrl_human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(vors_2020_ctrl_human, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(vors_2020_ctrl_human, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(vors_2020_ctrl_human, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(vors_2020_ctrl_human) ** 0.5), metric='cosine')\n",
    "tk.tl.triku(vors_2020_ctrl_human, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc.tl.umap(vors_2020_ctrl_human, min_dist=0.6, random_state=seed)\n",
    "sc.tl.leiden(vors_2020_ctrl_human, resolution=3, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(vors_2020_ctrl_human, dict_cats=dict_cats_fb_human, min_score=0.5, quantile_gene_sel=0.95)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_fb_human.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(vors_2020_ctrl_human, color=['assigned_cats'] + [i for i in val if i in vors_2020_ctrl_human.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(vors_2020_ctrl_human, color=['PDGFRA', 'LUM', 'DCN', 'COL1A1'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(vors_2020_ctrl_human, color=['Internal sample identifier', 'assigned_cats', 'leiden'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "vors_2020_ctrl_human_fb = vors_2020_ctrl_human[vors_2020_ctrl_human.obs['assigned_cats'].isin(['fibro'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(vors_2020_ctrl_human_fb, min_counts=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(vors_2020_ctrl_human_fb, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(vors_2020_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(vors_2020_ctrl_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(len(vors_2020_ctrl_human_fb) ** 0.5 // 5), metric='cosine')\n",
    "tk.tl.triku(vors_2020_ctrl_human_fb, use_raw=False)\n",
    "\n",
    "sc.pp.pca(vors_2020_ctrl_human_fb, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(vors_2020_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(vors_2020_ctrl_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(len(vors_2020_ctrl_human_fb) ** 0.5 // 5), metric='cosine')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(vors_2020_ctrl_human_fb, min_dist=0.4, random_state=seed)\n",
    "sc.tl.leiden(vors_2020_ctrl_human_fb, resolution=3, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(vors_2020_ctrl_human_fb, dict_cats=dict_cats_clusters, min_score=0.45, quantile_gene_sel=0.7, key_added='cluster', others_name='U')\n",
    "assign_cats(vors_2020_ctrl_human_fb, column_groupby='cluster', dict_cats=dict_cats_axes, min_score=0.4, key_added='axis', \n",
    "            intermediate_states=True, diff=0.15, others_name='U')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "vors_2020_ctrl_human_fb.uns['cluster_colors'] = [dict_colors_human[i] if i in dict_colors_human else '#bcbcbc' for \n",
    "                                      i in sorted(set(vors_2020_ctrl_human_fb.obs['cluster']))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(vors_2020_ctrl_human_fb, color=['Internal sample identifier', 'leiden', 'axis', 'cluster'], legend_loc='on data', cmap=magma, use_raw=False, )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "del vors_2020_ctrl_human_fb.obs['C3']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_clusters.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(vors_2020_ctrl_human_fb, color=['cluster'] + [i for i in val if i in vors_2020_ctrl_human_fb.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot_score_graph(vors_2020_ctrl_human_fb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "clear_adata(vors_2020_ctrl_human_fb)\n",
    "vors_2020_ctrl_human_fb.write_h5ad(vors_dir + '/vors_2020_ctrl_human_fb_processed.h5')\n",
    "vors_2020_ctrl_human.write_h5ad(vors_dir + '/vors_2020_ctrl_human_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "vors_2020_ctrl_human_fb = sc.read(vors_dir + '/vors_2020_ctrl_human_fb_processed.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Vorstandlechner et al. 2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vors_2021_dir = data_dir + '/Vorstandlechner_2021'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vors_2021_ctrl_human = sc.read(vors_2021_dir + '/vorstandlechner_2021_ctrl_human.h5', cache=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vors_2021_ctrl_human.var_names = [dict_rep[i] if i in dict_rep else i for i in vors_2021_ctrl_human.var_names]\n",
    "vors_2021_ctrl_human.var['gene_symbol'] = vors_2021_ctrl_human.var_names\n",
    "vors_2021_ctrl_human.var['Ensemble ID'] = [dict_biomart_human[i] if i in dict_biomart_human else 'NA' for i in vors_2021_ctrl_human.var_names]\n",
    "\n",
    "vors_2021_ctrl_human.raw = vors_2021_ctrl_human.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(vors_2021_ctrl_human, min_counts=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Basic QC filtering\n",
    "vors_2021_ctrl_human.var['mt'] = vors_2021_ctrl_human.var_names.str.startswith('MT-')  # annotate the group of mitochondrial genes as 'mt'\n",
    "sc.pp.calculate_qc_metrics(vors_2021_ctrl_human, qc_vars=['mt'], percent_top=None, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.violin(vors_2021_ctrl_human, ['n_genes_by_counts', 'total_counts', 'pct_counts_mt'],\n",
    "             jitter=0.4, multi_panel=True)\n",
    "\n",
    "sc.pl.scatter(vors_2021_ctrl_human, x='total_counts', y='pct_counts_mt')\n",
    "sc.pl.scatter(vors_2021_ctrl_human, x='total_counts', y='n_genes_by_counts')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1, 1, figsize=(10, 4))\n",
    "df = pd.DataFrame({'x': vors_2021_ctrl_human.obs['Internal sample identifier'], 'y': vors_2021_ctrl_human.obs['log1p_n_genes_by_counts']})\n",
    "sns.violinplot(x='x', y='y', data=df, ax=ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1, 1, figsize=(10, 4))\n",
    "df = pd.DataFrame({'x': vors_2021_ctrl_human.obs['Internal sample identifier'], 'y': vors_2021_ctrl_human.obs['pct_counts_mt']})\n",
    "sns.violinplot(x='x', y='y', data=df, ax=ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "vors_2021_ctrl_human = vors_2021_ctrl_human[(((vors_2021_ctrl_human.obs['Internal sample identifier'] == 'human_skin_1') & (vors_2021_ctrl_human.obs.log1p_n_genes_by_counts < 6) & \n",
    "                        (vors_2021_ctrl_human.obs.log1p_n_genes_by_counts > 4.9)) | \n",
    "                      ((vors_2021_ctrl_human.obs['Internal sample identifier'] == 'human_skin_2') & (vors_2021_ctrl_human.obs.log1p_n_genes_by_counts < 7.5) & \n",
    "                        (vors_2021_ctrl_human.obs.log1p_n_genes_by_counts > 6.3)) | \n",
    "                      ((vors_2021_ctrl_human.obs['Internal sample identifier'] == 'human_skin_3') & (vors_2021_ctrl_human.obs.log1p_n_genes_by_counts < 7.7) & \n",
    "                        (vors_2021_ctrl_human.obs.log1p_n_genes_by_counts > 6.3))).values, :]\n",
    "vors_2021_ctrl_human = vors_2021_ctrl_human[vors_2021_ctrl_human.obs.pct_counts_mt < 10, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(vors_2021_ctrl_human, min_counts=1)\n",
    "sc.pp.normalize_total(vors_2021_ctrl_human)\n",
    "sc.pp.log1p(vors_2021_ctrl_human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(vors_2021_ctrl_human, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(vors_2021_ctrl_human, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(vors_2021_ctrl_human, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(vors_2021_ctrl_human) ** 0.5), metric='cosine')\n",
    "tk.tl.triku(vors_2021_ctrl_human, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc.tl.umap(vors_2021_ctrl_human, min_dist=0.6, random_state=seed)\n",
    "sc.tl.leiden(vors_2021_ctrl_human, resolution=3, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(vors_2021_ctrl_human, dict_cats=dict_cats_fb_human, min_score=0.5, quantile_gene_sel=0.95)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(vors_2021_ctrl_human, color=['PDGFRA', 'LUM', 'DCN', 'COL1A1'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_fb_human.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(vors_2021_ctrl_human, color=['assigned_cats'] + [i for i in val if i in vors_2021_ctrl_human.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(vors_2021_ctrl_human, color=['Internal sample identifier', 'assigned_cats', 'leiden'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "vors_2021_ctrl_human_fb = vors_2021_ctrl_human[vors_2021_ctrl_human.obs['assigned_cats'].isin(['fibro'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(vors_2021_ctrl_human_fb, min_counts=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(vors_2021_ctrl_human_fb, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(vors_2021_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(vors_2021_ctrl_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(len(vors_2021_ctrl_human_fb) ** 0.5 // 5), metric='cosine')\n",
    "tk.tl.triku(vors_2021_ctrl_human_fb, use_raw=False)\n",
    "\n",
    "sc.pp.pca(vors_2021_ctrl_human_fb, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(vors_2021_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(vors_2021_ctrl_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(len(vors_2021_ctrl_human_fb) ** 0.5 // 5), metric='cosine')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(vors_2021_ctrl_human_fb, min_dist=0.4, random_state=seed)\n",
    "sc.tl.leiden(vors_2021_ctrl_human_fb, resolution=3, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(vors_2021_ctrl_human_fb, dict_cats=dict_cats_clusters, min_score=0.45, quantile_gene_sel=0.7, key_added='cluster', others_name='U')\n",
    "assign_cats(vors_2021_ctrl_human_fb, column_groupby='cluster', dict_cats=dict_cats_axes, min_score=0.4, key_added='axis', \n",
    "            intermediate_states=True, diff=0.15, others_name='U')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "vors_2021_ctrl_human_fb.uns['cluster_colors'] = [dict_colors_human[i] if i in dict_colors_human else '#bcbcbc' for \n",
    "                                      i in sorted(set(vors_2021_ctrl_human_fb.obs['cluster']))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(vors_2021_ctrl_human_fb, color=['Internal sample identifier', 'leiden', 'axis', 'cluster'], legend_loc='on data', cmap=magma, use_raw=False, )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "del vors_2021_ctrl_human_fb.obs['C3']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_clusters.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(vors_2021_ctrl_human_fb, color=['cluster'] + [i for i in val if i in vors_2021_ctrl_human_fb.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot_score_graph(vors_2021_ctrl_human_fb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "clear_adata(vors_2021_ctrl_human_fb)\n",
    "vors_2021_ctrl_human_fb.write_h5ad(vors_2021_dir + '/vors_2021_ctrl_human_fb_processed.h5')\n",
    "vors_2021_ctrl_human.write_h5ad(vors_2021_dir + '/vors_2021_ctrl_human_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "vors_2021_ctrl_human_fb = sc.read(vors_2021_dir + '/vors_2021_ctrl_human_fb_processed.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Xu et al. 2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "xu_2021_dir = data_dir + '/xu_2021'\n",
    "os.makedirs(xu_2021_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "xu_2021_ctrl_human = sc.read(xu_2021_dir + '/xu_2021_healthy.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "xu_2021_ctrl_human.var_names = [dict_rep[i] if i in dict_rep else i for i in xu_2021_ctrl_human.var_names]\n",
    "xu_2021_ctrl_human.var['gene_symbol'] = xu_2021_ctrl_human.var_names\n",
    "xu_2021_ctrl_human.var['Ensemble ID'] = [dict_biomart_human[i] if i in dict_biomart_human else 'NA' for i in xu_2021_ctrl_human.var_names]\n",
    "\n",
    "xu_2021_ctrl_human.raw = xu_2021_ctrl_human.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Basic QC filtering\n",
    "xu_2021_ctrl_human.var['mt'] = xu_2021_ctrl_human.var_names.str.startswith('MT-')  # annotate the group of mitochondrial genes as 'mt'\n",
    "sc.pp.calculate_qc_metrics(xu_2021_ctrl_human, qc_vars=['mt'], percent_top=None, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.violin(xu_2021_ctrl_human, ['n_genes_by_counts', 'total_counts', 'pct_counts_mt'],\n",
    "             jitter=0.4, multi_panel=True)\n",
    "\n",
    "sc.pl.scatter(xu_2021_ctrl_human, x='total_counts', y='pct_counts_mt')\n",
    "sc.pl.scatter(xu_2021_ctrl_human, x='total_counts', y='n_genes_by_counts')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1, 1, figsize=(10, 4))\n",
    "df = pd.DataFrame({'x': xu_2021_ctrl_human.obs['Internal sample identifier'], 'y': xu_2021_ctrl_human.obs['log1p_n_genes_by_counts']})\n",
    "sns.violinplot(x='x', y='y', data=df, ax=ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "xu_2021_ctrl_human = xu_2021_ctrl_human[(((xu_2021_ctrl_human.obs['Internal sample identifier'] == 'H01') & (xu_2021_ctrl_human.obs.log1p_n_genes_by_counts < 8) & \n",
    "                                                (xu_2021_ctrl_human.obs.log1p_n_genes_by_counts > 7.2)) | \n",
    "                                              ((xu_2021_ctrl_human.obs['Internal sample identifier'] == 'H02') & (xu_2021_ctrl_human.obs.log1p_n_genes_by_counts < 8) & \n",
    "                                                (xu_2021_ctrl_human.obs.log1p_n_genes_by_counts > 7.5)) | \n",
    "                                              ((xu_2021_ctrl_human.obs['Internal sample identifier'] == 'H03') & (xu_2021_ctrl_human.obs.log1p_n_genes_by_counts < 8) & \n",
    "                                                (xu_2021_ctrl_human.obs.log1p_n_genes_by_counts > 7.3) |\n",
    "                                              ((xu_2021_ctrl_human.obs['Internal sample identifier'] == 'H04') & (xu_2021_ctrl_human.obs.log1p_n_genes_by_counts < 8) & \n",
    "                                                (xu_2021_ctrl_human.obs.log1p_n_genes_by_counts > 7.2)) | \n",
    "                                              ((xu_2021_ctrl_human.obs['Internal sample identifier'] == 'H05') & (xu_2021_ctrl_human.obs.log1p_n_genes_by_counts < 8) & \n",
    "                                                (xu_2021_ctrl_human.obs.log1p_n_genes_by_counts > 7.2))) \n",
    "                                             ).values, :]\n",
    "xu_2021_ctrl_human = xu_2021_ctrl_human[xu_2021_ctrl_human.obs.pct_counts_mt < 7, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(xu_2021_ctrl_human, min_counts=1)\n",
    "sc.pp.normalize_total(xu_2021_ctrl_human)\n",
    "sc.pp.log1p(xu_2021_ctrl_human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(xu_2021_ctrl_human, random_state=seed, n_comps=30)\n",
    "sce.pp.harmony_integrate(xu_2021_ctrl_human, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(xu_2021_ctrl_human, use_rep='X_pca_harmony',  n_neighbors=int(0.5 * len(xu_2021_ctrl_human) ** 0.5 // 2), metric='cosine')\n",
    "tk.tl.triku(xu_2021_ctrl_human, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(xu_2021_ctrl_human, min_dist=0.4, random_state=seed)\n",
    "sc.tl.leiden(xu_2021_ctrl_human, resolution=5, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.subsample(xu_2021_ctrl_human, fraction=1, random_state=0, copy=False)\n",
    "sc.pl.umap(xu_2021_ctrl_human, color=['leiden', 'Internal sample identifier'], legend_loc='on data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(xu_2021_ctrl_human, color=['leiden', 'LUM', 'PDGFRA', 'COL1A1', 'DCN', 'RGS5', 'MYH11', 'MLANA', 'PMEL', 'HBB'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(xu_2021_ctrl_human, dict_cats=dict_cats_fb_human, min_score=0.45, quantile_gene_sel=0.95)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(xu_2021_ctrl_human, color=['leiden', 'assigned_cats'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(xu_2021_ctrl_human, color=['leiden', 'COL18A1', 'APCDD1', 'SLPI', 'CCL19', 'GGT5',  'CHRDL1', 'GPX3', 'BGN', 'ASPN', 'TNN', 'COL11A1', 'COCH', \n",
    "                                     'IGFBP3', 'ANGPTL7', 'SCN7A', 'C2orf40', 'NGFR', 'CLDN1', 'SBSPON', 'FGFBP2', 'DIO3', 'LUZP2'], legend_loc='on data', cmap=magma, use_raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_fb_human.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(xu_2021_ctrl_human, color=['assigned_cats'] + [i for i in val if i in xu_2021_ctrl_human.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "xu_2021_ctrl_human_fb = xu_2021_ctrl_human[xu_2021_ctrl_human.obs['assigned_cats'].isin(['fibro'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.filter_genes(xu_2021_ctrl_human_fb, min_counts=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pp.pca(xu_2021_ctrl_human_fb, random_state=seed, n_comps=50)\n",
    "sce.pp.harmony_integrate(xu_2021_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(xu_2021_ctrl_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(len(xu_2021_ctrl_human_fb) ** 0.5), metric='cosine')\n",
    "tk.tl.triku(xu_2021_ctrl_human_fb, use_raw=False)\n",
    "\n",
    "sc.pp.pca(xu_2021_ctrl_human_fb, random_state=seed, n_comps=50)\n",
    "sce.pp.harmony_integrate(xu_2021_ctrl_human_fb, key='Internal sample identifier', max_iter_harmony=50)\n",
    "sc.pp.neighbors(xu_2021_ctrl_human_fb, use_rep='X_pca_harmony',  n_neighbors=int(len(xu_2021_ctrl_human_fb) ** 0.5), metric='cosine')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.tl.umap(xu_2021_ctrl_human_fb, min_dist=0.5, random_state=seed)\n",
    "sc.tl.leiden(xu_2021_ctrl_human_fb, resolution=7, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assign_cats(xu_2021_ctrl_human_fb, dict_cats=dict_cats_clusters, min_score=0.5, quantile_gene_sel=0.85, key_added='cluster', others_name='U')\n",
    "assign_cats(xu_2021_ctrl_human_fb, column_groupby='cluster', dict_cats=dict_cats_axes, min_score=0.5, quantile_gene_sel=0.9,  key_added='axis', \n",
    "             intermediate_states=True, diff=0.15, others_name='U')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "xu_2021_ctrl_human_fb.uns['cluster_colors'] = [dict_colors_human[i] if i in dict_colors_human else '#bcbcbc' for \n",
    "                                      i in sorted(set(xu_2021_ctrl_human_fb.obs['cluster']))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc.pl.umap(xu_2021_ctrl_human_fb, color=['Internal sample identifier', 'leiden', 'axis', 'cluster'], legend_loc='on data', cmap=magma, use_raw=False, ncols=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "del xu_2021_ctrl_human_fb.obs['C3']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key, val in dict_cats_clusters.items():\n",
    "    print(key)\n",
    "    sc.pl.umap(xu_2021_ctrl_human_fb, color=['cluster'] + [i for i in val if i in xu_2021_ctrl_human_fb.var_names], legend_loc='on data', cmap=magma, use_raw=False, ncols=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot_score_graph(xu_2021_ctrl_human_fb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "clear_adata(xu_2021_ctrl_human_fb)\n",
    "xu_2021_ctrl_human_fb.write_h5ad(xu_2021_dir + '/xu_2021_ctrl_human_fb_processed.h5')\n",
    "xu_2021_ctrl_human.write_h5ad(xu_2021_dir + '/xu_2021_ctrl_human_processed.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "xu_2021_ctrl_human_fb = sc.read(xu_2021_dir + '/xu_2021_ctrl_human_fb_processed.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Presence of clusters for each dataset\n",
    "In this representation we will exclude Kim 2021, because being nail it is interesting to study for another case, but not exactly here."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from fb_functions import plot_adata_cluster_properties"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# The structure of the dataset dict is dict: [Name, Status (healthy, young, psoriasis, etc), year, ]\n",
    "list_datasets = [ahlers_2022_young_human_fb, billi_2022_ctrl_human_fb, boothby_2021_ctrl_human_fb, \n",
    "                 deng_2021_scar_human_fb, gao_2021_ctrl_human_fb, gaydosik_2020_ctrl_human_fb, gur_2022_ctrl_human_fb,\n",
    "                 he_2020_ctrl_human_fb, hughes_2020_ctrl_human_fb, kim_2020_ctrl_human_fb, \n",
    "                 liu_2021_ctrl_human_fb, mariottoni_2021_ctrl_human_fb, \n",
    "                 mirizio_2020_scl_human_fb, reynolds_2021_ctrl_human_fb, rindler_2021_ctrl_human_fb,\n",
    "                 sole_2020_young_human_fb, tabib_2018_ctrl_human_fb, tabib_2021_ctrl_human_fb, tsc_2021_ctrl_human_fb,\n",
    "                 theo_2020_ctrl_human_dm_fb, theo_2021_ctrl_human_fb,\n",
    "                 vors_2020_ctrl_human_fb, vors_2021_ctrl_human_fb, xu_2021_ctrl_human_fb]\n",
    "\n",
    "list_names = [adatax.obs['Author'].values[0] for adatax in list_datasets]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot_adata_cluster_properties(dict_cats_clusters=dict_cats_clusters, list_datasets=list_datasets, what='presence', cluster_name='cluster', axis_name='axis')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot_adata_cluster_properties(dict_cats_clusters=dict_cats_clusters, list_datasets=list_datasets, what='percentage', cluster_name='cluster', axis_name='axis')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot_adata_cluster_properties(dict_cats_clusters=dict_cats_clusters, list_datasets=list_datasets, what='axis', cluster_name='cluster', axis_name='axis')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plotting all Adatas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "fig, axs = plt.subplots(5, 5, figsize=(5 * 4, 5 * 4))\n",
    "\n",
    "for ax in axs.ravel()[len(list_datasets) - len(axs) :]:\n",
    "    ax.set_axis_off()\n",
    "\n",
    "for adata, name, idx in zip(list_datasets, list_names, range(len(list_datasets))):\n",
    "    sc.pl.umap(adata, color=['cluster'], legend_loc='on data', show=False, ax = axs.ravel()[idx], \n",
    "               title=str(adata.obs['Author'].iloc[0]) + ' ' + str(int(adata.obs['Year'].iloc[0])), size=15, cmap=magma, frameon=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PAGA "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "fig, axs = plt.subplots(5, 5, figsize=(5 * 4, 5 * 4))\n",
    "\n",
    "for ax in axs.ravel()[len(list_datasets) - len(axs) :]:\n",
    "    ax.set_axis_off()\n",
    "    \n",
    "for adata, name, idx in zip(list_datasets, list_names, range(len(list_datasets))):\n",
    "    sc.tl.paga(adata, groups='cluster')\n",
    "    sc.pl.paga(adata, ax=axs.ravel()[idx], frameon=False, show=False, \n",
    "               title=str(adata.obs['Author'].iloc[0]) + ' ' + str(int(adata.obs['Year'].iloc[0])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "fig, axs = plt.subplots(5, 5, figsize=(5 * 4, 5 * 4))\n",
    "\n",
    "for ax in axs.ravel()[len(list_datasets) - len(axs) :]:\n",
    "    ax.set_axis_off()\n",
    "    \n",
    "for adata, name, idx in zip(list_datasets, list_names, range(len(list_datasets))):\n",
    "    sc.tl.paga(adata, groups='cluster')\n",
    "    sc.pl.paga(adata, ax=axs.ravel()[idx], frameon=False, show=False, solid_edges='connectivities_tree', \n",
    "               title=str(adata.obs['Author'].iloc[0]) + ' ' + str(int(adata.obs['Year'].iloc[0])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "os.system('jupyter nbconvert --to html 2H_comparison_human_populations.ipynb')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:triku-notebooks]",
   "language": "python",
   "name": "conda-env-triku-notebooks-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "384px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}