0H_data_extraction_human.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "religious-seafood",
   "metadata": {},
   "source": [
    "# Data extraction\n",
    "\n",
    "In this notebook we are going to extract the raw or processed data from all datasets, to later analyze it in different notebooks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4e09cf62-807b-467c-ba07-0b87c44fc983",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "micro-currency",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "import mygene\n",
    "import numpy as np\n",
    "import os\n",
    "import pandas as pd\n",
    "import scanpy as sc\n",
    "import scanpy.external as sce\n",
    "import scipy.sparse as spr\n",
    "from tqdm import tqdm\n",
    "import triku as tk\n",
    "pd.set_option('display.max_columns', None)\n",
    "\n",
    "import requests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dcf67bbb-74b8-45cb-ab50-c7cd84e371c8",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from cellassign import assign_cats\n",
    "from fb_functions import metadata_assignment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9d89ffd1-4827-447b-8825-12f53628cd78",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Palettes for UMAP gene expression\n",
    "\n",
    "magma = [plt.get_cmap('magma')(i) for i in np.linspace(0,1, 80)]\n",
    "magma[0] = (0.88, 0.88, 0.88, 1)\n",
    "magma = mpl.colors.LinearSegmentedColormap.from_list(\"\", magma[:65])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "latin-spoke",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "data_dir = os.getcwd() + '/data/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "outdoor-jewelry",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "human_gencode_dir = \"/media/seth/SETH_DATA/SETH_Alex/Programs/human_GRCh38_gencode.v31.600\"\n",
    "program_dir = \"/media/seth/SETH_DATA/SETH_Alex/Programs/\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ab3c0096-3e45-44af-b720-b2d80d0f42a4",
   "metadata": {},
   "source": [
    "## Ahlers et al. 2022"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "16abe3ab-a263-495f-9bf8-6b5ee750a934",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "ahlers_2022_dir = data_dir + '/ahlers_2022'\n",
    "os.makedirs(ahlers_2022_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a2cf374c-b2c7-4c3e-94a7-888050bf469b",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {ahlers_2022_dir} && parallel-fastq-dump -s SRR15440580 --gzip --split-files -t 8 \n",
    "!cd {ahlers_2022_dir} && parallel-fastq-dump -s SRR15440581 --gzip --split-files -t 8\n",
    "!cd {ahlers_2022_dir} && parallel-fastq-dump -s SRR15440582 --gzip --split-files -t 8\n",
    "!cd {ahlers_2022_dir} && parallel-fastq-dump -s SRR15440583 --gzip --split-files -t 8 \n",
    "!cd {ahlers_2022_dir} && parallel-fastq-dump -s SRR15440584 --gzip --split-files -t 8\n",
    "!cd {ahlers_2022_dir} && parallel-fastq-dump -s SRR15440585 --gzip --split-files -t 8\n",
    "!cd {ahlers_2022_dir} && parallel-fastq-dump -s SRR15440586 --gzip --split-files -t 8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9cd0cdc-9748-4b45-b7bd-9006f20c627c",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame({'name': ['Ahlers_2022_P1_Y', 'Ahlers_2022_P2_O', 'Ahlers_2022_P3_Y', \n",
    "                            'Ahlers_2022_P4_O', 'Ahlers_2022_P5_Y', 'Ahlers_2022_P6_O', \n",
    "                            'Ahlers_2022_P7_O'], 'technology': ['10xv2'] * 7, \n",
    "                   'targetnumcells': [1000] * 7})\n",
    "df.to_csv(ahlers_2022_dir + '/metadata.tab', sep='\\t', index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d72e8479-3fa3-4c77-8449-b8bb37f80bc8",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!mv {ahlers_2022_dir}/SRR15440580_1.fastq.gz {ahlers_2022_dir}/Ahlers_2022_P4_O_L001_R1_001.fastq.gz \n",
    "!mv {ahlers_2022_dir}/SRR15440580_2.fastq.gz {ahlers_2022_dir}/Ahlers_2022_P4_O_L001_R2_001.fastq.gz \n",
    "\n",
    "!mv {ahlers_2022_dir}/SRR15440581_1.fastq.gz {ahlers_2022_dir}/Ahlers_2022_P3_Y_L001_R1_001.fastq.gz \n",
    "!mv {ahlers_2022_dir}/SRR15440581_2.fastq.gz {ahlers_2022_dir}/Ahlers_2022_P3_Y_L001_R2_001.fastq.gz \n",
    "\n",
    "!mv {ahlers_2022_dir}/SRR15440582_1.fastq.gz {ahlers_2022_dir}/Ahlers_2022_P2_O_L001_R1_001.fastq.gz \n",
    "!mv {ahlers_2022_dir}/SRR15440582_2.fastq.gz {ahlers_2022_dir}/Ahlers_2022_P2_O_L001_R2_001.fastq.gz \n",
    "\n",
    "!mv {ahlers_2022_dir}/SRR15440583_1.fastq.gz {ahlers_2022_dir}/Ahlers_2022_P1_Y_L001_R1_001.fastq.gz \n",
    "!mv {ahlers_2022_dir}/SRR15440583_2.fastq.gz {ahlers_2022_dir}/Ahlers_2022_P1_Y_L001_R2_001.fastq.gz \n",
    "\n",
    "!mv {ahlers_2022_dir}/SRR15440584_1.fastq.gz {ahlers_2022_dir}/Ahlers_2022_P7_O_L001_R1_001.fastq.gz \n",
    "!mv {ahlers_2022_dir}/SRR15440584_2.fastq.gz {ahlers_2022_dir}/Ahlers_2022_P7_O_L001_R2_001.fastq.gz \n",
    "\n",
    "!mv {ahlers_2022_dir}/SRR15440585_1.fastq.gz {ahlers_2022_dir}/Ahlers_2022_P6_O_L001_R1_001.fastq.gz \n",
    "!mv {ahlers_2022_dir}/SRR15440585_2.fastq.gz {ahlers_2022_dir}/Ahlers_2022_P6_O_L001_R2_001.fastq.gz \n",
    "\n",
    "!mv {ahlers_2022_dir}/SRR15440586_1.fastq.gz {ahlers_2022_dir}/Ahlers_2022_P5_Y_L001_R1_001.fastq.gz \n",
    "!mv {ahlers_2022_dir}/SRR15440586_2.fastq.gz {ahlers_2022_dir}/Ahlers_2022_P5_Y_L001_R2_001.fastq.gz "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "15e27de3-dbd4-4d32-a9b8-b5d1e9151c5d",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {ahlers_2022_dir} && loompy fromfq Ahlers_2022_P1_Y.loom Ahlers_2022_P1_Y {human_gencode_dir} metadata.tab \\\n",
    "Ahlers_2022_P1_Y_L001_R1_001.fastq.gz Ahlers_2022_P1_Y_L001_R2_001.fastq.gz "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c72e64c2-de35-43c5-9998-0c9a1e055b20",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!cd {ahlers_2022_dir} && loompy fromfq Ahlers_2022_P2_O.loom Ahlers_2022_P2_O {human_gencode_dir} metadata.tab \\\n",
    "Ahlers_2022_P2_O_L001_R1_001.fastq.gz Ahlers_2022_P2_O_L001_R2_001.fastq.gz "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ad42e34b-b74e-4867-8eae-15a7ffda53c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {ahlers_2022_dir} && loompy fromfq Ahlers_2022_P3_Y.loom Ahlers_2022_P3_Y {human_gencode_dir} metadata.tab \\\n",
    "Ahlers_2022_P3_Y_L001_R1_001.fastq.gz Ahlers_2022_P3_Y_L001_R2_001.fastq.gz "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "97931f1c-0db5-41db-94e2-f0201824fd8a",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {ahlers_2022_dir} && loompy fromfq Ahlers_2022_P4_O.loom Ahlers_2022_P4_O {human_gencode_dir} metadata.tab \\\n",
    "Ahlers_2022_P4_O_L001_R1_001.fastq.gz Ahlers_2022_P4_O_L001_R2_001.fastq.gz "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "44400077-f189-4827-b798-f3a6806c18d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {ahlers_2022_dir} && loompy fromfq Ahlers_2022_P5_Y.loom Ahlers_2022_P5_Y {human_gencode_dir} metadata.tab \\\n",
    "Ahlers_2022_P5_Y_L001_R1_001.fastq.gz Ahlers_2022_P5_Y_L001_R2_001.fastq.gz "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "09f2a0de-952d-43f8-9ca1-89ae960d4e62",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {ahlers_2022_dir} && loompy fromfq Ahlers_2022_P6_O.loom Ahlers_2022_P6_O {human_gencode_dir} metadata.tab \\\n",
    "Ahlers_2022_P6_O_L001_R1_001.fastq.gz Ahlers_2022_P6_O_L001_R2_001.fastq.gz "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "15348cf9-b966-4d82-82b9-01934dc00989",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {ahlers_2022_dir} && loompy fromfq Ahlers_2022_P7_O.loom Ahlers_2022_P7_O {human_gencode_dir} metadata.tab \\\n",
    "Ahlers_2022_P7_O_L001_R1_001.fastq.gz Ahlers_2022_P7_O_L001_R2_001.fastq.gz "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9f175fa0-7f26-4be6-b156-1400e9e7210e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_Ahlers_2022_P1_Y = sc.read_loom(ahlers_2022_dir + '/Ahlers_2022_P1_Y.loom')\n",
    "adata_Ahlers_2022_P1_Y.var_names_make_unique()\n",
    "metadata_assignment(adata_Ahlers_2022_P1_Y, 'Ahlers', 2022, 'P1_Y')\n",
    "\n",
    "adata_Ahlers_2022_P2_O = sc.read_loom(ahlers_2022_dir + '/Ahlers_2022_P2_O.loom')\n",
    "adata_Ahlers_2022_P2_O.var_names_make_unique()\n",
    "metadata_assignment(adata_Ahlers_2022_P2_O, 'Ahlers', 2022, 'P2_O')\n",
    "\n",
    "adata_Ahlers_2022_P3_Y = sc.read_loom(ahlers_2022_dir + '/Ahlers_2022_P3_Y.loom')\n",
    "adata_Ahlers_2022_P3_Y.var_names_make_unique()\n",
    "metadata_assignment(adata_Ahlers_2022_P3_Y, 'Ahlers', 2022, 'P3_Y')\n",
    "\n",
    "adata_Ahlers_2022_P4_O = sc.read_loom(ahlers_2022_dir + '/Ahlers_2022_P4_O.loom')\n",
    "adata_Ahlers_2022_P4_O.var_names_make_unique()\n",
    "metadata_assignment(adata_Ahlers_2022_P4_O, 'Ahlers', 2022, 'P4_O')\n",
    "\n",
    "adata_Ahlers_2022_P5_Y = sc.read_loom(ahlers_2022_dir + '/Ahlers_2022_P5_Y.loom')\n",
    "adata_Ahlers_2022_P5_Y.var_names_make_unique()\n",
    "metadata_assignment(adata_Ahlers_2022_P5_Y, 'Ahlers', 2022, 'P5_Y')\n",
    "\n",
    "adata_Ahlers_2022_P6_O = sc.read_loom(ahlers_2022_dir + '/Ahlers_2022_P6_O.loom')\n",
    "adata_Ahlers_2022_P6_O.var_names_make_unique()\n",
    "metadata_assignment(adata_Ahlers_2022_P6_O, 'Ahlers', 2022, 'P6_O')\n",
    "\n",
    "adata_Ahlers_2022_P7_O = sc.read_loom(ahlers_2022_dir + '/Ahlers_2022_P7_O.loom')\n",
    "adata_Ahlers_2022_P7_O.var_names_make_unique()\n",
    "metadata_assignment(adata_Ahlers_2022_P7_O, 'Ahlers', 2022, 'P7_O')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c1fa1406-ed64-4dc6-b43b-53145bda8a5a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_Ahlers_2022_young = sc.AnnData.concatenate(adata_Ahlers_2022_P1_Y, adata_Ahlers_2022_P3_Y, adata_Ahlers_2022_P5_Y, \n",
    "                                             batch_key=\"Internal sample identifier\", batch_categories=['P1_Y', 'P3_Y', 'P5_Y'])\n",
    "\n",
    "adata_Ahlers_2022_old = sc.AnnData.concatenate(adata_Ahlers_2022_P2_O, adata_Ahlers_2022_P4_O, adata_Ahlers_2022_P6_O, adata_Ahlers_2022_P7_O,\n",
    "                                             batch_key=\"Internal sample identifier\", batch_categories=['P2_Y', 'P4_Y', 'P6_Y', 'P7_Y'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7ba3d11f-8b23-47c0-94b0-1be1fba77275",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_Ahlers_2022_young.write_h5ad(ahlers_2022_dir + '/ahlers_2022_young_human.h5')\n",
    "adata_Ahlers_2022_old.write_h5ad(ahlers_2022_dir + '/ahlers_2022_old_human.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "da7aa645-a09e-42a7-a518-e19c57b3415d",
   "metadata": {},
   "source": [
    "## Alkon et al. 2022 (human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8ad5450c-3b7e-462f-81db-2616a862460d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "alkon_2022_dir = data_dir + '/alkon_2022'\n",
    "os.makedirs(alkon_2022_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "be4ac489-4563-4ed6-a9f3-607cd47d7883",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for gse, sample in zip(['GSM5474333', 'GSM5474334', 'GSM5474335', 'GSM5474336', 'GSM5474337', 'GSM5474338', 'GSM5474339'], ['NHS1', 'NHS2', 'NHS3', 'AD2', 'AD3', 'AD4', 'AD5']):\n",
    "    !cd {alkon_2022_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/{gse[:7]}nnn/{gse}/suppl/{gse}%5F{sample}%5Fbarcodes%2Etsv%2Egz\n",
    "    !cd {alkon_2022_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/{gse[:7]}nnn/{gse}/suppl/{gse}%5F{sample}%5Ffeatures%2Etsv%2Egz\n",
    "    !cd {alkon_2022_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/{gse[:7]}nnn/{gse}/suppl/{gse}%5F{sample}%5Fmatrix%2Emtx%2Egz\n",
    "        \n",
    "    !mkdir {alkon_2022_dir}/{sample}\n",
    "    !mv {alkon_2022_dir}/{gse}_{sample}_barcodes.tsv.gz {alkon_2022_dir}/{sample}/barcodes.tsv.gz\n",
    "    !mv {alkon_2022_dir}/{gse}_{sample}_features.tsv.gz {alkon_2022_dir}/{sample}/features.tsv.gz\n",
    "    !mv {alkon_2022_dir}/{gse}_{sample}_matrix.mtx.gz {alkon_2022_dir}/{sample}/matrix.mtx.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eb4f09d0-c4c4-49b7-9835-99487d4abc3e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "alkon_2022_healthy_1 = sc.read_10x_mtx(alkon_2022_dir + '/NHS1')\n",
    "metadata_assignment(alkon_2022_healthy_1, 'Alkon', 2022, 'NHS1')\n",
    "\n",
    "alkon_2022_healthy_2 = sc.read_10x_mtx(alkon_2022_dir + '/NHS2')\n",
    "metadata_assignment(alkon_2022_healthy_2, 'Alkon', 2022, 'NHS2')\n",
    "\n",
    "alkon_2022_healthy_3 = sc.read_10x_mtx(alkon_2022_dir + '/NHS3')\n",
    "metadata_assignment(alkon_2022_healthy_3, 'Alkon', 2022, 'NHS3')\n",
    "\n",
    "alkon_2022_healthy_human = sc.AnnData.concatenate(alkon_2022_healthy_1, alkon_2022_healthy_2, alkon_2022_healthy_3, \n",
    "                                         batch_categories=['NHS1', 'NHS2', 'NHS3'], batch_key='Internal sample identifier')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0bc4407d-d1eb-4aef-ae7e-22f451fd633c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "alkon_2022_healthy_human.write_h5ad(alkon_2022_dir + '/alkon_2022_ctrl_human.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "51b24fbb-4a27-41b5-8daa-588ad8f0b7e8",
   "metadata": {},
   "source": [
    "## Billi et al. 2022 (human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ea237461-3feb-4197-bfd4-b500a1afca2b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "billi_2022_dir = data_dir + '/billi_2022'\n",
    "os.makedirs(billi_2022_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "232d1fe3-82db-4283-9014-70b9a21b899e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "list_gsm = ['GSM' + str(i) for i in range(5652699, 5652713)]\n",
    "list_names = ['0' + str(i) for i in range(1,10)] + [str(i) for i in range(10, 15)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd17dac3-36ed-47f3-9832-a762f74340ae",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for gsm, name in zip(list_gsm, list_names):\n",
    "    !cd {billi_2022_dir} && wget ftp://ftp.ncbi.nlm.nih.gov/geo/samples/GSM5652nnn/{gsm}/suppl/{gsm}%5FHealthy%5FD{name}%5Fcounts%2Etar%2Egz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b159810d-9270-4f5e-8d66-cdf428b1f248",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!cd {billi_2022_dir} && cat *.tar.gz | tar zxvf - -i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ef8fbfc9-583d-4e36-9930-10eef63eb768",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "list_adatas = []\n",
    "for gsm, name in zip(list_gsm, list_names):\n",
    "    adata = sc.read_10x_mtx(billi_2022_dir + f'/Healthy_D{name}_counts')\n",
    "    adata = metadata_assignment(adata, 'Billi', 2022, 'H' + str(int(name)), do_return=True)\n",
    "    list_adatas.append(adata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "429ee81f-abfb-4dad-ab53-f421bbdc7cc6",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_HC = sc.AnnData.concatenate(*list_adatas, \n",
    "                                  batch_key='Internal sample identifier', \n",
    "                                  batch_categories=['H' + i for i in list_names]\n",
    "                                   )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "16291b7c-7768-43c8-bb59-00c2b5af9cb1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_HC.X = spr.csr_matrix(adata_HC.X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d78339ef-a659-43b7-8696-7035d84a8ba3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_HC.write_h5ad(billi_2022_dir + '/billi_2022_ctrl_human.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e21ebad5-9915-4f34-a16c-afd7f71ce8f6",
   "metadata": {},
   "source": [
    "## Burja et al. 2022 (human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9c476baf-9137-4d73-89e0-53deef67408a",
   "metadata": {},
   "outputs": [],
   "source": [
    "burja_2022_dir = data_dir + '/burja_2022'\n",
    "os.makedirs(burja_2022_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d706865-eaa8-4231-9084-25d97c752d5e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!cd {burja_2022_dir} && wget ftp://ftp.sra.ebi.ac.uk/vol1/run/ERR912/ERR9121019/F1_o25154_1_5-HC_fresh_S5_L001_R1.fastq.gz\n",
    "!cd {burja_2022_dir} && wget ftp://ftp.sra.ebi.ac.uk/vol1/run/ERR912/ERR9121019/F1_o25154_1_5-HC_fresh_S5_L001_R2.fastq.gz\n",
    "!cd {burja_2022_dir} && wget ftp://ftp.sra.ebi.ac.uk/vol1/run/ERR912/ERR9121020/F1_o25154_1_5-HC_fresh_S5_L002_R1.fastq.gz\n",
    "!cd {burja_2022_dir} && wget ftp://ftp.sra.ebi.ac.uk/vol1/run/ERR912/ERR9121020/F1_o25154_1_5-HC_fresh_S5_L002_R2.fastq.gz\n",
    "!cd {burja_2022_dir} && wget ftp://ftp.sra.ebi.ac.uk/vol1/run/ERR912/ERR9121021/F2_o25459_1_6-HS_Blaz_S12_L002_R1.fastq.gz\n",
    "!cd {burja_2022_dir} && wget ftp://ftp.sra.ebi.ac.uk/vol1/run/ERR912/ERR9121021/F2_o25459_1_6-HS_Blaz_S12_L002_R2.fastq.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "05eb0f8f-b2c3-4315-9e71-2c47d82f882f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df = pd.DataFrame({'name': ['F1_o25154_1_5-HC_fresh_S5', 'F2_o25459_1_6-HS_Blaz_S12'], 'technology': ['10xv3'] * 2, 'targetnumcells': [5000] * 2})\n",
    "\n",
    "df.to_csv(burja_2022_dir + '/metadata.tab', sep='\\t', index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "23b405e0-0f29-437c-87f8-fec50ba1c304",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!cd {burja_2022_dir} && loompy AnnData F1_o25154_1_5-HC_fresh_S5.loom F1_o25154_1_5-HC_fresh_S5 {human_gencode_dir} metadata.tab \\\n",
    "F1_o25154_1_5-HC_fresh_S5_L001_R1.fastq.gz F1_o25154_1_5-HC_fresh_S5_L001_R2.fastq.gz F1_o25154_1_5-HC_fresh_S5_L002_R1.fastq.gz F1_o25154_1_5-HC_fresh_S5_L002_R2.fastq.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "71018430-3779-4ea0-8fa1-1f04b9e7e3ba",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!cd {burja_2022_dir} && loompy fromfq F2_o25459_1_6-HS_Blaz_S12.loom F2_o25459_1_6-HS_Blaz_S12 {human_gencode_dir} metadata.tab \\\n",
    "F2_o25459_1_6-HS_Blaz_S12_L002_R1.fastq.gz F2_o25459_1_6-HS_Blaz_S12_L002_R2.fastq.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "118c4db0-df71-4efa-8ff7-26dad1ccde2f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_burja_fresh_1 = sc.read(burja_2022_dir + '/F1_o25154_1_5-HC_fresh_S5.loom')\n",
    "adata_burja_fresh_1 = metadata_assignment(adata_burja_fresh_1, 'Burja', 2022, 'HC 1', do_return=True)\n",
    "adata_burja_fresh_1.var_names_make_unique()\n",
    "\n",
    "adata_burja_fresh_2 = sc.read(burja_2022_dir + '/F2_o25459_1_6-HS_Blaz_S12.loom')\n",
    "adata_burja_fresh_2 = metadata_assignment(adata_burja_fresh_2, 'Burja', 2022, 'HC 2', do_return=True)\n",
    "adata_burja_fresh_2.var_names_make_unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "edd5c3ae-b1c9-4a68-a535-98cd586b5311",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "burja_2022_ctrl_human = sc.AnnData.concatenate(adata_burja_fresh_1, adata_burja_fresh_2, \n",
    "                                   batch_key=\"Internal sample identifier\", batch_categories=['HC 1', 'HC 2'])\n",
    "sc.pp.filter_cells(burja_2022_ctrl_human, min_counts=10)\n",
    "burja_2022_ctrl_human.write_h5ad(burja_2022_dir + '/burja_2022_ctrl_human.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "89d14a47-6f49-46e6-9b4c-c2a5bbfebb51",
   "metadata": {
    "toc-hr-collapsed": true
   },
   "source": [
    "## Boothby et al. 2021 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cefb74a6-c68c-4b46-9136-f28fdd16e04e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "boothby_2021_dir = data_dir + '/boothby_2021'\n",
    "os.makedirs(boothby_2021_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "06ba109f-8240-44d0-b681-f4e7e47796ec",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!cd {boothby_2021_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM5549nnn/GSM5549903/suppl/GSM5549903%5FHC01%5Ffiltered%5Ffeature%5Fbc%5Fmatrix%2Eh5\n",
    "!cd {boothby_2021_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM5549nnn/GSM5549904/suppl/GSM5549904%5FHC02%5Ffiltered%5Ffeature%5Fbc%5Fmatrix%2Eh5\n",
    "!cd {boothby_2021_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM5549nnn/GSM5549905/suppl/GSM5549905%5FHC03%5Ffiltered%5Ffeature%5Fbc%5Fmatrix%2Eh5\n",
    "!cd {boothby_2021_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM5549nnn/GSM5549906/suppl/GSM5549906%5FEF01%5Ffiltered%5Ffeature%5Fbc%5Fmatrix%2Eh5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "25454e4c-2307-428a-a6bd-e9abe603b8ea",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "boothby_2021_HC01 = sc.read_10x_h5(boothby_2021_dir + '/GSM5549903_HC01_filtered_feature_bc_matrix.h5')\n",
    "boothby_2021_HC01.var_names_make_unique()\n",
    "metadata_assignment(boothby_2021_HC01, 'Boothby', 2021, 'HC01')\n",
    "\n",
    "boothby_2021_HC02 = sc.read_10x_h5(boothby_2021_dir + '/GSM5549904_HC02_filtered_feature_bc_matrix.h5')\n",
    "boothby_2021_HC02.var_names_make_unique()\n",
    "metadata_assignment(boothby_2021_HC02, 'Boothby', 2021, 'HC02')\n",
    "\n",
    "boothby_2021_HC03 = sc.read_10x_h5(boothby_2021_dir + '/GSM5549905_HC03_filtered_feature_bc_matrix.h5')\n",
    "boothby_2021_HC03.var_names_make_unique()\n",
    "metadata_assignment(boothby_2021_HC03, 'Boothby', 2021, 'HC03')\n",
    "\n",
    "boothby_2021_EF01 = sc.read_10x_h5(boothby_2021_dir + '/GSM5549906_EF01_filtered_feature_bc_matrix.h5')\n",
    "boothby_2021_EF01.var_names_make_unique()\n",
    "metadata_assignment(boothby_2021_EF01, 'Boothby', 2021, 'EF01')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f4ba7874-86cd-4beb-bc36-da5be88a8462",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "boothby_2021_ctrl_human = sc.AnnData.concatenate(boothby_2021_HC01, boothby_2021_HC02, boothby_2021_HC03, \n",
    "                                   batch_key=\"Internal sample identifier\", batch_categories=['HC01', 'HC02', 'HC03'])\n",
    "boothby_2021_ctrl_human.write_h5ad(boothby_2021_dir + '/boothby_2021_ctrl_human.h5')\n",
    "\n",
    "boothby_2021_EF01.write_h5ad(boothby_2021_dir + '/boothby_2021_ef_human.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "afa95bba-b5de-4a98-be60-f787aa71fca5",
   "metadata": {
    "toc-hr-collapsed": true
   },
   "source": [
    "## Capolupo et al. 2022 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2594b633-ff46-4dda-a048-7da1994afa81",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "capolupo_2022_dir = data_dir + '/capolupo_2022'\n",
    "os.makedirs(capolupo_2022_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "959ec360-581a-485b-8a88-17c4331967e3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!cd {capolupo_2022_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM5099nnn/GSM5099588/suppl/GSM5099588%5Fraw%5Ffeature%5Fbc%5Fmatrix%5Fcontrol2%2Etar%2Egz\n",
    "!cd {capolupo_2022_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM5099nnn/GSM5099586/suppl/GSM5099586%5Fraw%5Ffeature%5Fbc%5Fmatrix%5Fcontrol1%2Etar%2Egz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "75e60016-c8a9-43a6-a858-bae28b3a053b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!cd {capolupo_2022_dir} && tar xzvf GSM5099588_raw_feature_bc_matrix_control2.tar.gz\n",
    "!cd {capolupo_2022_dir} && mv raw_feature_bc_matrix raw_feature_bc_matrix_2\n",
    "!cd {capolupo_2022_dir} && tar xzvf GSM5099586_raw_feature_bc_matrix_control1.tar.gz\n",
    "!cd {capolupo_2022_dir} && mv raw_feature_bc_matrix raw_feature_bc_matrix_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e2ab6ad3-1a3f-4a49-ab04-d7797a85085e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "capolupo_2022_ctrl_human_rep1 = sc.read_10x_mtx(capolupo_2022_dir + '/raw_feature_bc_matrix_1')\n",
    "capolupo_2022_ctrl_human_rep1.var_names_make_unique()\n",
    "metadata_assignment(capolupo_2022_ctrl_human_rep1, 'Capolupo', 2022, 'In vitro dHF replica 1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7340a181-c4e7-4468-9c47-386ef963a0c1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "capolupo_2022_ctrl_human_rep2 = sc.read_10x_mtx(capolupo_2022_dir + '/raw_feature_bc_matrix_2')\n",
    "capolupo_2022_ctrl_human_rep2.var_names_make_unique()\n",
    "metadata_assignment(capolupo_2022_ctrl_human_rep2, 'Capolupo', 2022, 'In vitro dHF replica 2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "91aa52a8-27a2-4985-8d40-9abf18491914",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "capolupo_2022_ctrl_human = sc.AnnData.concatenate(capolupo_2022_ctrl_human_rep1, capolupo_2022_ctrl_human_rep2, \n",
    "                                   batch_key=\"Internal sample identifier\", batch_categories=['HC01', 'HC02', ])\n",
    "sc.pp.filter_cells(capolupo_2022_ctrl_human, min_counts=150)\n",
    "capolupo_2022_ctrl_human.write_h5ad(capolupo_2022_dir + '/capolupo_2022_ctrl_human.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9e383877-4dfd-4220-9a28-359a6d213f4d",
   "metadata": {},
   "source": [
    "## Deng et al. 2021 (human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "014c6592-2581-451a-84ee-0d46eca52a96",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "deng_dir = data_dir + '/deng_2021'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0511310e-cb7c-4762-9cd0-124bb234b1e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {deng_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4994nnn/GSM4994382/suppl/GSM4994382_NS1_matrix.tar.gz\n",
    "!cd {deng_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4994nnn/GSM4994383/suppl/GSM4994383_NS2_matrix.tar.gz\n",
    "!cd {deng_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4994nnn/GSM4994384/suppl/GSM4994384_NS3_matrix.tar.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "de28d946-cb39-46eb-b4f7-8aa6f4add0c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {deng_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4994nnn/GSM4994379/suppl/GSM4994379_KL1_matrix.tar.gz\n",
    "!cd {deng_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4994nnn/GSM4994380/suppl/GSM4994380_KL2_matrix.tar.gz\n",
    "!cd {deng_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4994nnn/GSM4994381/suppl/GSM4994381_KL3_matrix.tar.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "246c11c3-7bef-49c3-8c16-e9f7a57cc4d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {deng_dir} && cat *.tar.gz | tar zxvf - -i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f04967f3-4c46-426d-a4c3-30161dba2423",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_deng_scar_1 = sc.read_10x_mtx(deng_dir + '/NF1_matrix')\n",
    "metadata_assignment(adata_deng_scar_1, 'Deng', 2021, 'Scar_1')\n",
    "\n",
    "adata_deng_scar_2 = sc.read_10x_mtx(deng_dir + '/NF2_matrix')\n",
    "metadata_assignment(adata_deng_scar_2, 'Deng', 2021, 'Scar_2')\n",
    "\n",
    "\n",
    "adata_deng_scar_3 = sc.read_10x_mtx(deng_dir + '/NF3_matrix')\n",
    "metadata_assignment(adata_deng_scar_3, 'Deng', 2021, 'Scar_3')\n",
    "\n",
    "adata_deng_scar = sc.AnnData.concatenate(adata_deng_scar_1, adata_deng_scar_2, adata_deng_scar_3, \n",
    "                                         batch_categories=['Scar_1', 'Scar_2', 'Scar_3'], batch_key='Internal sample identifier')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a8d5b5d7-d69a-48ae-8f71-c8e33c15e202",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_deng_keloid_1 = sc.read_10x_mtx(deng_dir + '/NF1_matrix')\n",
    "metadata_assignment(adata_deng_keloid_1, 'Deng', 2021, 'Keloid_1')\n",
    "\n",
    "adata_deng_keloid_2 = sc.read_10x_mtx(deng_dir + '/NF2_matrix')\n",
    "metadata_assignment(adata_deng_keloid_2, 'Deng', 2021, 'Keloid_2')\n",
    "\n",
    "\n",
    "adata_deng_keloid_3 = sc.read_10x_mtx(deng_dir + '/NF3_matrix')\n",
    "metadata_assignment(adata_deng_keloid_3, 'Deng', 2021, 'Keloid_3')\n",
    "\n",
    "adata_deng_keloid = sc.AnnData.concatenate(adata_deng_keloid_1, adata_deng_keloid_2, adata_deng_keloid_3, \n",
    "                                         batch_categories=['Keloid_1', 'Keloid_2', 'Keloid_3'], batch_key='Internal sample identifier')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5fbbb853-751c-4737-9c27-29696bbc778d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_deng_scar.write_h5ad(deng_dir + '/deng_2021_scar_human.h5')\n",
    "adata_deng_keloid.write_h5ad(deng_dir + '/deng_2021_keloid_human.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ae7c1902-b506-4080-9095-10f4730698b5",
   "metadata": {},
   "source": [
    "## Gao et al. 2021 (human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bae78e15-8cd4-4bfc-bb88-65ff7ad4931a",
   "metadata": {},
   "outputs": [],
   "source": [
    "gao_dir = data_dir + '/gao_2021'\n",
    "os.makedirs(gao_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e729330b-a991-42d3-9f24-e778e6d3469d",
   "metadata": {},
   "source": [
    "### Direct h5ad download"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9279a3ed-86a5-406d-8bff-a1c25295e209",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "!aria2c -x 16 https://ftp.ncbi.nlm.nih.gov/geo/series/GSE162nnn/GSE162183/suppl/GSE162183%5FRaw%5Fgene%5Fcounts%5Fmatrix%5FLoomFile%2Eloom%2Egz -d {gao_dir} -o gao_2021.loom.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d34d094e-5b6f-4a7b-bd01-7dd8faf887d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "!gunzip {gao_dir}/gao_2021.loom.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6f3f2eff-8360-447e-884b-d2175de4c8dd",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_gao = sc.read_loom(f'{gao_dir}/gao_2021.loom')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "19f171f0-3070-4141-b895-9ebd4240d87c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_gao_ctrl1 = metadata_assignment(adata_gao[adata_gao.obs['Patient'] == 'Ctrl1'], 'Gao', 2021, 'Ctrl1', do_return=True)\n",
    "adata_gao_ctrl2 = metadata_assignment(adata_gao[adata_gao.obs['Patient'] == 'Ctrl2'], 'Gao', 2021, 'Ctrl2', do_return=True)\n",
    "adata_gao_ctrl3 = metadata_assignment(adata_gao[adata_gao.obs['Patient'] == 'Ctrl3'], 'Gao', 2021, 'Ctrl3', do_return=True)\n",
    "\n",
    "adata_gao_psor1 = metadata_assignment(adata_gao[adata_gao.obs['Patient'] == 'Psor1'], 'Gao', 2021, 'Psor1', do_return=True)\n",
    "adata_gao_psor2 = metadata_assignment(adata_gao[adata_gao.obs['Patient'] == 'Psor2'], 'Gao', 2021, 'Psor2', do_return=True)\n",
    "adata_gao_psor3 = metadata_assignment(adata_gao[adata_gao.obs['Patient'] == 'Psor3'], 'Gao', 2021, 'Psor3', do_return=True)\n",
    "\n",
    "adata_gao_ctrl = sc.AnnData.concatenate(adata_gao_ctrl1, adata_gao_ctrl2, adata_gao_ctrl3, \n",
    "                                         batch_categories=['Ctrl1', 'Ctrl2', 'Ctrl3'], batch_key='Internal sample identifier')\n",
    "\n",
    "adata_gao_psor = sc.AnnData.concatenate(adata_gao_psor1, adata_gao_psor2, adata_gao_psor3, \n",
    "                                         batch_categories=['Psor1', 'Psor2', 'Psor3'], batch_key='Internal sample identifier')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e9b216e2-659c-4680-855e-eba0323bdc4b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_gao_ctrl.write_h5ad(gao_dir + '/gao_2021_ctrl_human.h5')\n",
    "adata_gao_psor.write_h5ad(gao_dir + '/gao_2021_psor_human.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ab648631-188c-4065-9cf3-346d67f7c64e",
   "metadata": {},
   "source": [
    "## Gaydosik et al. 2020 (human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a41999fc-74ec-422d-ac95-6afbcf8b215a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "gaydosik_dir = data_dir + '/gaydosik_2020'\n",
    "os.makedirs(gaydosik_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4376ec7c-045e-407a-9d9e-fce2e2853d03",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "!cd {gaydosik_dir} && wget ftp://ftp.ncbi.nlm.nih.gov/geo/samples/GSM3679nnn/GSM3679033/suppl/GSM3679033%5FLabeled%5FSC67%5F050517%5FSK%5FMF2%5FGRCh38raw%2Ecsv%2Egz\n",
    "!cd {gaydosik_dir} && wget ftp://ftp.ncbi.nlm.nih.gov/geo/samples/GSM3679nnn/GSM3679034/suppl/GSM3679034%5FLabeled%5FSC82%5F060617%5FSK%5FMF5%5FGRCh38raw%2Ecsv%2Egz\n",
    "!cd {gaydosik_dir} && wget ftp://ftp.ncbi.nlm.nih.gov/geo/samples/GSM3679nnn/GSM3679035/suppl/GSM3679035%5FSC157dataframe%2Ecsv%2Egz\n",
    "!cd {gaydosik_dir} && wget ftp://ftp.ncbi.nlm.nih.gov/geo/samples/GSM3679nnn/GSM3679036/suppl/GSM3679036%5FSC158dataframe%2Ecsv%2Egz\n",
    "!cd {gaydosik_dir} && wget ftp://ftp.ncbi.nlm.nih.gov/geo/samples/GSM3679nnn/GSM3679037/suppl/GSM3679037%5FSC205dataframe%2Ecsv%2Egz\n",
    "\n",
    "!cd {gaydosik_dir} && wget ftp://ftp.ncbi.nlm.nih.gov/geo/samples/GSM3679nnn/GSM3679038/suppl/GSM3679038%5FLabeled%5FSC50%5F011917%5FSK%5FNOR%5FGRCh38raw%2Ecsv%2Egz\n",
    "!cd {gaydosik_dir} && wget ftp://ftp.ncbi.nlm.nih.gov/geo/samples/GSM3679nnn/GSM3679039/suppl/GSM3679039%5FLabeled%5FSC68%5F051517%5FSK%5FNOR%5FGRCh38raw%2Ecsv%2Egz\n",
    "!cd {gaydosik_dir} && wget ftp://ftp.ncbi.nlm.nih.gov/geo/samples/GSM3679nnn/GSM3679040/suppl/GSM3679040%5FLabeled%5FSC124%5F080317%5FSK%5FNOR%5FGRCh38raw%2Ecsv%2Egz\n",
    "!cd {gaydosik_dir} && wget ftp://ftp.ncbi.nlm.nih.gov/geo/samples/GSM3679nnn/GSM3679041/suppl/GSM3679041%5FLabeled%5FSC125%5F080317%5FSK%5FNOR%5FGRCh38raw%2Ecsv%2Egz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ca29dc5-de71-4eba-9419-d9a19cdfe03f",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {gaydosik_dir} &&  gunzip *.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a4f384c3-f3e3-4d4f-96bc-6f63d0cc4be6",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_CTCL2 = sc.read(gaydosik_dir + '/GSM3679033_Labeled_SC67_050517_SK_MF2_GRCh38raw.csv').transpose()\n",
    "adata_CTCL2 = metadata_assignment(adata_CTCL2, 'Gaydosik', 2019, 'CTCL2', do_return=True)\n",
    "\n",
    "adata_CTCL5 = sc.read(gaydosik_dir + '/GSM3679034_Labeled_SC82_060617_SK_MF5_GRCh38raw.csv').transpose()\n",
    "adata_CTCL5 = metadata_assignment(adata_CTCL5, 'Gaydosik', 2019, 'CTCL5', do_return=True)\n",
    "\n",
    "adata_CTCL6 = sc.read(gaydosik_dir + '/GSM3679035_SC157dataframe.csv').transpose()\n",
    "adata_CTCL6 = metadata_assignment(adata_CTCL6, 'Gaydosik', 2019, 'CTCL6', do_return=True)\n",
    "\n",
    "adata_CTCL8 = sc.read(gaydosik_dir + '/GSM3679036_SC158dataframe.csv').transpose()\n",
    "adata_CTCL8 = metadata_assignment(adata_CTCL8, 'Gaydosik', 2019, 'CTCL8', do_return=True)\n",
    "\n",
    "adata_CTCL12 = sc.read(gaydosik_dir + '/GSM3679037_SC205dataframe.csv').transpose()\n",
    "adata_CTCL12 = metadata_assignment(adata_CTCL12, 'Gaydosik', 2019, 'CTCL12', do_return=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e4be092a-c163-48fd-ac83-f72feb53e336",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_HC1 = sc.read(gaydosik_dir + '/GSM3679038_Labeled_SC50_011917_SK_NOR_GRCh38raw.csv').transpose()\n",
    "adata_HC1 = metadata_assignment(adata_HC1, 'Gaydosik', 2019, 'HC1', do_return=True)\n",
    "\n",
    "adata_HC2 = sc.read(gaydosik_dir + '/GSM3679039_Labeled_SC68_051517_SK_NOR_GRCh38raw.csv').transpose()\n",
    "adata_HC2 = metadata_assignment(adata_HC2, 'Gaydosik', 2019, 'HC1', do_return=True)\n",
    "\n",
    "adata_HC3 = sc.read(gaydosik_dir + '/GSM3679040_Labeled_SC124_080317_SK_NOR_GRCh38raw.csv').transpose()\n",
    "adata_HC3 = metadata_assignment(adata_HC3, 'Gaydosik', 2019, 'HC1', do_return=True)\n",
    "\n",
    "adata_HC4 = sc.read(gaydosik_dir + '/GSM3679041_Labeled_SC125_080317_SK_NOR_GRCh38raw.csv').transpose()\n",
    "adata_HC4 = metadata_assignment(adata_HC4, 'Gaydosik', 2019, 'HC1', do_return=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ec7bfef8-a286-4449-9c5c-3c80ace70d27",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_CTCL = sc.AnnData.concatenate(adata_CTCL2, adata_CTCL5, adata_CTCL6, \n",
    "                                   adata_CTCL8, adata_CTCL12, batch_key='Internal sample identifier', \n",
    "                                   batch_categories=['CTCL2', 'CTCL5', 'CTCL6',\n",
    "                                                     'CTCL8', 'CTCL12'])\n",
    "adata_HC = sc.AnnData.concatenate(adata_HC1, adata_HC2, adata_HC3, \n",
    "                                   adata_HC4, batch_key='Internal sample identifier', batch_categories=[\n",
    "                                       'HC1', 'HC2', 'HC3', 'HC4'\n",
    "                                   ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6bf7bd5a-7616-48bf-8ab1-7a55d84d5e5c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_CTCL.X = spr.csr_matrix(adata_CTCL.X)\n",
    "adata_HC.X = spr.csr_matrix(adata_HC.X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "09fe7c61-64f0-49dd-aeda-f9436715f9cf",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_CTCL.write_h5ad(gaydosik_dir + '/gaydosik_2020_CTCL_human.h5')\n",
    "adata_HC.write_h5ad(gaydosik_dir + '/gaydosik_2020_ctrl_human.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9096027d-8bc0-477d-a10e-0563daeb1378",
   "metadata": {},
   "source": [
    "## Gür et al. 2022 (human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "305a9ba6-54ba-4f3f-815d-6770b0e35582",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "gur_2022_dir = data_dir + '/gur_2022'\n",
    "os.makedirs(gur_2022_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f24cadde-cfa0-4bc3-b2ca-3e5070ccd515",
   "metadata": {},
   "source": [
    "We need to get a table of gsm, patients and markers to select only the skin samples, since that's what we are interested in. Since there is not direct info on that, we will do some html scrapping."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "882b53b7-e7d7-43b1-bd80-fe0dfabc6394",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "list_gsm, list_tis, list_marker, list_pat, list_title = [], [], [], [], []\n",
    "\n",
    "for gsm in tqdm(['GSM' + str(i) for i in range(5836979, 5837701)]):\n",
    "    url = f\"https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc={gsm}\"\n",
    "    code = requests.get(url).text\n",
    "    \n",
    "    pos_pat = code.find('patient id: ') + 12\n",
    "    pos_tis = code.find('tissue: ') + 8\n",
    "    pos_marker = code.find('selection marker: ') + 18\n",
    "    pos_title = code.find(\"\"\"<tr valign=\"top\"><td nowrap>Title</td>\\n<td style=\"text-align: justify\">\"\"\") + 71\n",
    "\n",
    "    code_tissue = code[pos_tis:]\n",
    "    code_tissue = code_tissue[: code_tissue.find('<br>')]\n",
    "    \n",
    "    code_patient = code[pos_pat:]\n",
    "    code_patient = code_patient[: code_patient.find('<br>')]\n",
    "    \n",
    "    code_marker = code[pos_marker:]\n",
    "    code_marker = code_marker[: code_marker.find('<br>')]\n",
    "    \n",
    "    code_title = code[pos_title:]\n",
    "    code_title = code_title[: code_title.find('</td>')]\n",
    "    \n",
    "    list_tis.append(code_tissue)\n",
    "    list_marker.append(code_marker)\n",
    "    list_pat.append(code_patient)\n",
    "    list_gsm.append(gsm)\n",
    "    list_title.append(code_title)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "823724fa-09a0-4b6f-96b7-7573f63e66a0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_gsm = pd.DataFrame({'GSM': list_gsm, 'Tissue': list_tis, 'Marker': list_marker, 'Patient': list_pat, 'Title': list_title})\n",
    "df_gsm_sub = df_gsm[(df_gsm.Tissue == 'Skin') & (df_gsm.Marker == 'CD90+')]\n",
    "df_gsm_sub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "278f531d-fbfd-4009-ba89-2bd823cf4f13",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_gsm_sub_ctrl_human = df_gsm_sub.loc[[True if (i[:4] == 'Ctrl') & (len(i) == 7) else False for i in df_gsm_sub.Patient],:]\n",
    "df_gsm_sub_ctrl_human"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e190cf55-e9ba-40d6-b800-1a8283ccc3c6",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_gsm_sub_ssc = df_gsm_sub.loc[[True if (i[:2] == 'pt') & (len(i) == 7) else False for i in df_gsm_sub.Patient],:]\n",
    "df_gsm_sub_ssc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b0b3600a-352a-402e-8f88-4c236874ab39",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "list_GSM_sub_ctrl_human = df_gsm_sub_ctrl_human['GSM'].values\n",
    "list_title_sub_ctrl_human = df_gsm_sub_ctrl_human['Title'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f84119e8-2a22-40b6-993b-3bc08eb63f8b",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for GSM, title in tqdm(zip(list_GSM_sub_ctrl_human, list_title_sub_ctrl_human)):\n",
    "    !cd {gur_2022_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/{GSM[:7]}nnn/{GSM}/suppl/{GSM}%5f{title}%2Etxt%2Egz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5aef8186-8233-4208-a59e-bcf7c0921078",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "list_GSM_sub_ssc = df_gsm_sub_ssc['GSM'].values\n",
    "list_title_sub_ssc = df_gsm_sub_ssc['Title'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b024fb89-f201-4390-a804-5256e2c9a338",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for GSM, title in tqdm(zip(list_GSM_sub_ssc, list_title_sub_ssc)):\n",
    "    !cd {gur_2022_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/{GSM[:7]}nnn/{GSM}/suppl/{GSM}%5f{title}%2Etxt%2Egz"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "adcdb7f7-f2f0-4d26-abf2-731e64f37068",
   "metadata": {},
   "source": [
    "Creating the adatas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "930d3b65-ec58-4fdf-9fd0-3ddcb2595f10",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "list_adatas = []\n",
    "for GSM, title in tqdm(zip(list_GSM_sub_ctrl_human, list_title_sub_ctrl_human)):\n",
    "    adata = sc.read(gur_2022_dir + f\"/{GSM}_{title}.txt.gz\").transpose()\n",
    "    adata.var_names_make_unique()\n",
    "    metadata_assignment(adata, 'Gur', 2022, title)\n",
    "    list_adatas.append(adata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "25270eb2-5fb8-4c5f-8da8-3018022bf8cc",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "gur_2022_ctrl_human = sc.AnnData.concatenate(*list_adatas, batch_key=\"Internal sample identifier\", batch_categories=list_title_sub_ctrl_human)\n",
    "gur_2022_ctrl_human.write_h5ad(gur_2022_dir + '/gur_2022_ctrl_human.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a2f85538-5afb-4050-9522-79e9f508fe94",
   "metadata": {},
   "source": [
    "## He et al. 2020 (human)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "650da279-e11b-4ef3-ac9b-097696ef5925",
   "metadata": {},
   "source": [
    "### Raw data and metadata extraction (healthy samples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bef957da-71ff-4053-925a-290da80b92ea",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "he_dir = data_dir + '/He_2020'\n",
    "os.makedirs(he_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eb67505c-427a-4a57-91f1-8db44844ad54",
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -rf {he_dir}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "42af0693-e2ae-4af6-8eae-3f56b68c2fde",
   "metadata": {},
   "outputs": [],
   "source": [
    "SRA_list = \"\"\"\n",
    "SRR11396171\n",
    "SRR11396175\n",
    "SRR11396162\n",
    "SRR11396164\n",
    "SRR11396166\n",
    "SRR11396167\n",
    "SRR11396168\n",
    "SRR11396170\n",
    "\"\"\"\n",
    "\n",
    "with open(he_dir + '/accession.txt', 'w') as f:\n",
    "    f.write(SRA_list)\n",
    "    \n",
    "df = pd.DataFrame({'name': ['He2020'], 'technology': ['10xv2'] * 8, 'targetnumcells': [5000] * 8})\n",
    "\n",
    "df.to_csv(he_dir + '/metadata.tab', sep='\\t', index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f0759e57-1679-449c-af3f-758c4f098b2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {he_dir} && cat accession.txt | parallel -j 8 \"prefetch {}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "404c935c-7ca8-42d2-972d-a23eb3b14bc2",
   "metadata": {},
   "outputs": [],
   "source": [
    "def adapt_fastq(filename_dir, filename_root, idx):\n",
    "    # Using readline() \n",
    "    filein = open(f'{filename_dir}/{filename_root}.fastq', 'r') \n",
    "    \n",
    "    fileR1 = open(f'{filename_dir}/He2020_L00{idx}_R1_001.fastq', 'w') \n",
    "    fileR2 = open(f'{filename_dir}/He2020_L00{idx}_R2_001.fastq', 'w') \n",
    "    \n",
    "    print(f'{filename_dir}/{filename_root}.fastq', f'{filename_dir}/He2020_L00{idx}_R1_001.fastq')\n",
    "    \n",
    "    count = 0\n",
    "    \n",
    "    while True: \n",
    "        count += 1\n",
    "\n",
    "        # Get next line from file \n",
    "        line = filein.readline() \n",
    "        \n",
    "        if count % 4 in [1, 3]:\n",
    "            fileR1.write(line.replace('\\n', '') + '\\n')\n",
    "            fileR2.write(line.replace('\\n', '') + '\\n')\n",
    "        elif count == 2:\n",
    "            fileR1.write(line.replace('\\n', '')[:26] + '\\n')\n",
    "            fileR2.write(line.replace('\\n', '')[26:] + '\\n')\n",
    "        else:\n",
    "            fileR1.write(line.replace('\\n', '')[:26] + '\\n')\n",
    "            fileR2.write(line.replace('\\n', '')[26:] + '\\n')\n",
    "            \n",
    "\n",
    "        # if line is empty \n",
    "        # end of file is reached \n",
    "        if not line: \n",
    "            break\n",
    "\n",
    "    filein.close() \n",
    "    fileR1.close()\n",
    "    fileR2.close()\n",
    "    \n",
    "    os.system(f'gzip {filename_dir}/{filename_root}.fastq')\n",
    "    os.system(f'gzip {filename_dir}/He2020_L00{idx}_R1_001.fastq')\n",
    "    os.system(f'gzip {filename_dir}/He2020_L00{idx}_R2_001.fastq')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "15f5315e-c5e7-480c-971c-fd997c507e29",
   "metadata": {},
   "outputs": [],
   "source": [
    "adapt_fastq_remote = ray.remote(adapt_fastq)\n",
    "\n",
    "ray.init(ignore_reinit_error=True, num_cpus=2)\n",
    "\n",
    "ret = [adapt_fastq_remote.remote(f'{he_dir}', f'{name}', name_idx+1) for name_idx, name in enumerate(SRA_list.split('\\n')[1:-1])]\n",
    "ray.get(ret)\n",
    "\n",
    "ray.shutdown()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bf15720c-e35e-4937-9567-36a3b0e0f08c",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {he_dir} && loompy fromfq He2020.loom He2020 {human_gencode_dir} metadata.tab \\\n",
    "He2020_L001_R1_001.fastq.gz He2020_L001_R2_001.fastq.gz He2020_L002_R1_001.fastq.gz He2020_L002_R2_001.fastq.gz \\\n",
    "He2020_L003_R1_001.fastq.gz He2020_L003_R2_001.fastq.gz He2020_L004_R1_001.fastq.gz He2020_L004_R2_001.fastq.gz \\\n",
    "He2020_L005_R1_001.fastq.gz He2020_L005_R2_001.fastq.gz He2020_L006_R1_001.fastq.gz He2020_L006_R2_001.fastq.gz \\\n",
    "He2020_L007_R1_001.fastq.gz He2020_L007_R2_001.fastq.gz He2020_L008_R1_001.fastq.gz He2020_L008_R2_001.fastq.gz \\"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1f8a8787-e842-494c-8aed-30c659c09122",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_he_2020_ctrl_human = sc.read(he_dir + '/He2020.loom')\n",
    "adata_he_2020_ctrl_human = metadata_assignment(adata_he_2020_ctrl_human, 'He', 2020, 'Ctrl', do_return=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "98fa510b-8390-4673-9565-a39d74d1edb0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_he_2020_ctrl_human.write_h5ad(he_dir + '/adata_he_2020_ctrl_human.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e3ab479d-d8cb-42d6-9d2c-a14e3251678f",
   "metadata": {},
   "source": [
    "### Raw data and metadata extraction (lesional samples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "69ba93d5-f81e-4615-aeb6-910774e3ea0e",
   "metadata": {},
   "outputs": [],
   "source": [
    "SRA_list = \"\"\"\n",
    "SRR11396159\n",
    "SRR11396160\n",
    "SRR11396163\n",
    "SRR11396165\n",
    "\"\"\"\n",
    "\n",
    "with open(he_dir + '/accession_old.txt', 'w') as f:\n",
    "    f.write(SRA_list)\n",
    "    \n",
    "df = pd.DataFrame({'name': ['He2020_inj'], 'technology': ['10xv2'], 'targetnumcells': [5000]})\n",
    "\n",
    "df.to_csv(he_dir + '/metadata_inj.tab', sep='\\t', index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "78c74de4-2383-4a18-81f5-3c2c16621c00",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {he_dir} && cat accession_inj.txt | parallel -j 8 \"prefetch {}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "869abdc0-67c7-46d8-8976-8b6db4930b08",
   "metadata": {},
   "outputs": [],
   "source": [
    "adapt_fastq_remote = ray.remote(adapt_fastq)\n",
    "\n",
    "ray.init(ignore_reinit_error=True, num_cpus=2)\n",
    "\n",
    "ret = [adapt_fastq_remote.remote(f'{he_dir}', f'{name}', name_idx+10) for name_idx, name in enumerate(SRA_list.split('\\n')[1:-1])]\n",
    "ray.get(ret)\n",
    "\n",
    "ray.shutdown()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "794504ce-38a9-428e-a570-d725a80ce6ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {he_dir} && loompy fromfq He2020_inj.loom He2020_inj {human_gencode_dir} metadata_old.tab \\\n",
    "He2020_L0010_R1_001.fastq.gz He2020_L0010_R2_001.fastq.gz He2020_L0011_R1_001.fastq.gz He2020_L0011_R2_001.fastq.gz \\\n",
    "He2020_L0013_R1_001.fastq.gz He2020_L0013_R2_001.fastq.gz He2020_L0012_R1_001.fastq.gz He2020_L0012_R2_001.fastq.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "32ae8a9c-fd87-4d3a-b825-c1a5bb8dc088",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_he_2020_AD_LS = sc.read(he_dir + '/He2020_inj.loom')\n",
    "adata_he_2020_AD_LS = metadata_assignment(adata_he_2020_AD_LS, 'He', 2020, 'AD_LS', do_return=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "49ed07de-6547-4d8e-b00b-2bf5fcd0d94b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_he_2020_AD_LS.write_h5ad(he_dir + '/adata_he_2020_AD_LS_human.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2a4d2d8f-8b62-4138-83a3-70096f24f514",
   "metadata": {},
   "source": [
    "## Hughes et al. 2020 (human)\n",
    "\n",
    "SRA download and processing did not produce correct matrices."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ba29cea2-875d-44eb-bdb4-d2386a2fe2e3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "hughes_dir = data_dir + '/hughes_2020'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b9ee287c-3c64-41f5-b88f-bf46c4ba6ce9",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {hughes_dir} && wget https://storage.googleapis.com/fc-9fe89f6d-a673-4659-8332-0bdcecf88e03/alexandria_structured_metadata.txt?GoogleAccessId=116798894341-compute%40developer.gserviceaccount.com&Expires=1632753650&Signature=nqwIaNbwIJdKkgwebfZSwN69btGHB%2BSzDbhRbstLDwkbFjGs%2BRThwg1Ibicv7aRdjN1KvJt%2F3w6kH1wht7bpS19a5TDEG9O4vycd%2FpKLdReTTvc10tkNlkT%2FVFTkpZhfSb1AXKxWNVSQsNj4oF5L%2FrXZvHU4DVYSfGxKhFbIjccqdWi%2B04k0cuALBrHzaBQllnOw079Rur3L5TvobxizFvIV77ZlQZ2cUI8iOO1UQ7ZB70Gfdb%2Fxr6UVZHCfb%2FEAthgAHDVk%2FrNx%2FuPvmFb5XKwQEvLeoCzMQ80uixU8L4XHKBL1YSLDoaVDeIc6bjEt86%2Fl1e31Aymh0ERxvzEYrQ%3D%3D\n",
    "!cd {hughes_dir} && https://ftp.ncbi.nlm.nih.gov/geo/series/GSE150nnn/GSE150672/suppl/GSE150672%5FSkin%5FExpression%5Fcounts%2Ecsv%2Egz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9151961b-86bf-4e32-b68e-3160b128e44e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_hughes = sc.read(hughes_dir + '/GSE150672_Skin_Expression_counts.csv.gz').transpose()\n",
    "adata_hughes.var_names_make_unique()\n",
    "adata_hughes.X = spr.csr.csr_matrix(adata_hughes.X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3063b128-df79-48e7-9e5c-5b9bdc1ec9a7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "metadata_hughes = pd.read_csv(hughes_dir+'/alexandria_structured_metadata.txt', sep='\\t', index_col=0)\n",
    "metadata_hughes = metadata_hughes.iloc[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5568a759-b990-4b0b-8a54-b73120270ef6",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_hughes.obs.loc[metadata_hughes.index, ['donor_id']] = metadata_hughes.loc[:, ['donor_id']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c85345ad-cf67-4531-a06e-31356060ad37",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "list_adatas = [ ]\n",
    "\n",
    "list_donors = ['Acne1', 'Acne2', 'Acne3', 'Acne4', 'Alopecia1', 'GA1', 'GA2', 'Leprosy1', 'Leprosy2', 'Leprosy3', 'Leprosy4', 'Normal1', 'Normal2', 'Normal3', 'Psoriasis1', 'Psoriasis2', 'Psoriasis3', 'Psoriasis4', 'Psoriasis5',]\n",
    "for donor in list_donors:\n",
    "    adata_donor = adata_hughes[adata_hughes.obs['donor_id'] == donor]\n",
    "    adata_donor = metadata_assignment(adata_donor, 'Hughes', 2020, donor, do_return=True)\n",
    "    list_adatas.append(adata_donor)\n",
    "    \n",
    "adata_hughes_all = sc.AnnData.concatenate(*list_adatas, batch_key=\"Internal sample identifier\", batch_categories=list_donors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4043dc3b-d708-4d4f-9410-5ff47ac20896",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_hughes_all.write_h5ad(hughes_dir + '/adata_hughes_2020_all_human.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d9127062-977d-42c3-a992-6ac6d99e634a",
   "metadata": {},
   "source": [
    "## Kim et al. 2020 (human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "20ce387d-557a-4057-9d07-7789cf725803",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "kim_dir = data_dir + '/Kim_2020'\n",
    "os.makedirs(kim_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9b1ca538-1eab-4840-a70e-71a727150cbf",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {kim_dir} && fastq-dump SRR9307706 --gzip --split-files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e8df39af-4ba6-4f80-837b-6bc5e749c851",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {kim_dir} && fastq-dump SRR9307707 --gzip --split-files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "45c1f34d-8ca6-4d20-9000-fcb0fdbeedde",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {kim_dir} && fastq-dump SRR9307708 --gzip --split-files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "83f61131-745d-4b0e-9a6d-e831917c93bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {kim_dir} && fastq-dump SRR9307709 --gzip --split-files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5309da15-09a7-4ba7-b09b-1ce7469c95fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {kim_dir} && fastq-dump SRR9307710 --gzip --split-files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "904bf7ad-b885-4042-9e3c-05846e00aadd",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {kim_dir} && fastq-dump SRR9307711 --gzip --split-files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0068ff9c-55e6-4388-9c56-db3a5e53f6f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {kim_dir}/injury && fastq-dump SRR9307698 --gzip --split-files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aba5ee15-95db-4190-87b8-864f8031d33a",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame({'name': ['Kim_2020_HC1', 'Kim_2020_HC2', 'Kim_2020_HC3', \n",
    "                            'Kim_2020_HC4', 'Kim_2020_HC5', 'Kim_2020_HC6', \n",
    "                            'Kim_2020_inj'], 'technology': ['10xv2'] * 7, \n",
    "                   'targetnumcells': [1000] * 7})\n",
    "df.to_csv(kim_dir + '/metadata.tab', sep='\\t', index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bdb1e5e6-4df3-4f45-a312-4db4add8f40a",
   "metadata": {},
   "outputs": [],
   "source": [
    "!mv {kim_dir}/injury/SRR9307698_2.fastq.gz {kim_dir}/Kim_2020_inj_L001_R1_001.fastq.gz \n",
    "!mv {kim_dir}/injury/SRR9307698_3.fastq.gz {kim_dir}/Kim_2020_inj_L001_R2_001.fastq.gz \n",
    "\n",
    "!mv {kim_dir}/SRR9307706_2.fastq.gz {kim_dir}/Kim_2020_HC1_L001_R1_001.fastq.gz \n",
    "!mv {kim_dir}/SRR9307706_3.fastq.gz {kim_dir}/Kim_2020_HC1_L001_R2_001.fastq.gz \n",
    "!mv {kim_dir}/SRR9307707_2.fastq.gz {kim_dir}/Kim_2020_HC2_L001_R1_001.fastq.gz \n",
    "!mv {kim_dir}/SRR9307707_3.fastq.gz {kim_dir}/Kim_2020_HC2_L001_R2_001.fastq.gz \n",
    "!mv {kim_dir}/SRR9307708_2.fastq.gz {kim_dir}/Kim_2020_HC3_L001_R1_001.fastq.gz \n",
    "!mv {kim_dir}/SRR9307708_3.fastq.gz {kim_dir}/Kim_2020_HC3_L001_R2_001.fastq.gz \n",
    "!mv {kim_dir}/SRR9307709_2.fastq.gz {kim_dir}/Kim_2020_HC4_L001_R1_001.fastq.gz \n",
    "!mv {kim_dir}/SRR9307709_3.fastq.gz {kim_dir}/Kim_2020_HC4_L001_R2_001.fastq.gz \n",
    "!mv {kim_dir}/SRR9307710_2.fastq.gz {kim_dir}/Kim_2020_HC5_L001_R1_001.fastq.gz \n",
    "!mv {kim_dir}/SRR9307710_3.fastq.gz {kim_dir}/Kim_2020_HC5_L001_R2_001.fastq.gz \n",
    "!mv {kim_dir}/SRR9307711_2.fastq.gz {kim_dir}/Kim_2020_HC6_L001_R1_001.fastq.gz \n",
    "!mv {kim_dir}/SRR9307711_3.fastq.gz {kim_dir}/Kim_2020_HC6_L001_R2_001.fastq.gz "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "41891c15-3c25-46fd-a66d-2f22e704eac3",
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -rf {kim_dir}/*_1.fastq.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "abe6e70b-d931-4c66-a6dd-8822c4007cfa",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {kim_dir} && loompy fromfq Kim_2020_HC1.loom Kim_2020_HC1 {human_gencode_dir} metadata.tab \\\n",
    "Kim_2020_HC1_L001_R1_001.fastq.gz Kim_2020_HC1_L001_R2_001.fastq.gz "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4ed51f93-da4f-4685-837b-834d2eb68ceb",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {kim_dir} && loompy fromfq Kim_2020_HC2.loom Kim_2020_HC2 {human_gencode_dir} metadata.tab \\\n",
    "Kim_2020_HC2_L001_R1_001.fastq.gz Kim_2020_HC2_L001_R2_001.fastq.gz "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3861da4a-33fd-4408-bc6f-f241e0303df3",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {kim_dir} && loompy fromfq Kim_2020_HC3.loom Kim_2020_HC3 {human_gencode_dir} metadata.tab \\\n",
    "Kim_2020_HC3_L001_R1_001.fastq.gz Kim_2020_HC3_L001_R2_001.fastq.gz "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9b57cc4c-5995-489c-be67-695aaf52d7fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {kim_dir} && loompy fromfq Kim_2020_HC4.loom Kim_2020_HC4 {human_gencode_dir} metadata.tab \\\n",
    "Kim_2020_HC4_L001_R1_001.fastq.gz Kim_2020_HC4_L001_R2_001.fastq.gz "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ccb4ca1-a436-46c7-83d8-5aab15440e11",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {kim_dir} && loompy fromfq Kim_2020_HC5.loom Kim_2020_HC5 {human_gencode_dir} metadata.tab \\\n",
    "Kim_2020_HC5_L001_R1_001.fastq.gz Kim_2020_HC5_L001_R2_001.fastq.gz "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b525bfcf-e0cd-4f40-b9b7-0cdb0be95035",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {kim_dir} && loompy fromfq Kim_2020_HC6.loom Kim_2020_HC6 {human_gencode_dir} metadata.tab \\\n",
    "Kim_2020_HC6_L001_R1_001.fastq.gz Kim_2020_HC6_L001_R2_001.fastq.gz "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c27073c5-73ad-4398-a3df-3c4ca91d4aa6",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {kim_dir} && loompy fromfq Kim_2020_inj.loom Kim_2020_inj {human_gencode_dir} metadata.tab \\\n",
    "Kim_2020_inj_L001_R1_001.fastq.gz Kim_2020_inj_L001_R2_001.fastq.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8f741d62-59b9-405e-80ad-72c523439f04",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_kim_HC1 = sc.read_loom(kim_dir + '/Kim_2020_HC1.loom')\n",
    "adata_kim_HC1.var_names_make_unique()\n",
    "metadata_assignment(adata_kim_HC1, 'Kim', 2020, 'HC1')\n",
    "\n",
    "adata_kim_HC2 = sc.read_loom(kim_dir + '/Kim_2020_HC2.loom')\n",
    "adata_kim_HC2.var_names_make_unique()\n",
    "metadata_assignment(adata_kim_HC2, 'Kim', 2020, 'HC2')\n",
    "\n",
    "adata_kim_HC3 = sc.read_loom(kim_dir + '/Kim_2020_HC3.loom')\n",
    "adata_kim_HC3.var_names_make_unique()\n",
    "metadata_assignment(adata_kim_HC3, 'Kim', 2020, 'HC3')\n",
    "\n",
    "adata_kim_HC4 = sc.read_loom(kim_dir + '/Kim_2020_HC4.loom')\n",
    "adata_kim_HC4.var_names_make_unique()\n",
    "metadata_assignment(adata_kim_HC4, 'Kim', 2020, 'HC4')\n",
    "\n",
    "adata_kim_HC5 = sc.read_loom(kim_dir + '/Kim_2020_HC5.loom')\n",
    "adata_kim_HC5.var_names_make_unique()\n",
    "metadata_assignment(adata_kim_HC5, 'Kim', 2020, 'HC5')\n",
    "\n",
    "adata_kim_HC6 = sc.read_loom(kim_dir + '/Kim_2020_HC6.loom')\n",
    "adata_kim_HC6.var_names_make_unique()\n",
    "metadata_assignment(adata_kim_HC6, 'Kim', 2020, 'HC6')\n",
    "\n",
    "adata_kim_inj = sc.read_loom(kim_dir + '/Kim_2020_inj.loom')\n",
    "adata_kim_inj.var_names_make_unique()\n",
    "metadata_assignment(adata_kim_inj, 'Kim', 2020, 'inj')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "473b2083-f040-449a-bee9-0e0d9fbeec1e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_kim_2020_ctrl_human = sc.AnnData.concatenate(adata_kim_HC1, adata_kim_HC2, adata_kim_HC3, adata_kim_HC4, adata_kim_HC5, adata_kim_HC6, \n",
    "                                             batch_key=\"Internal sample identifier\", batch_categories=['HC1', 'HC2', 'HC3', 'HC4', 'HC5', 'HC6'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e371bff8-9597-4e78-be6f-aad111989a62",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_kim_2020_ctrl_human.write_h5ad(kim_dir + '/adata_kim_2020_ctrl_human.h5')\n",
    "adata_kim_inj.write_h5ad(kim_dir + '/adata_kim_2020_dress_human.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d7cf6a42-4b09-42e5-a1ee-99b43a356049",
   "metadata": {
    "toc-hr-collapsed": true
   },
   "source": [
    "## Kim et al. 2021 (human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "20661c97-fa53-453a-8b29-4c2f387ccb89",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "kim_dir = data_dir + '/kim_2021'\n",
    "os.makedirs(kim_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a86ca3fb-01d3-4281-a172-03352097043f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!cd {kim_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4816nnn/GSM4816776/suppl/GSM4816776%5FPoly01%5Fbarcodes%2Etsv%2Egz\n",
    "!cd {kim_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4816nnn/GSM4816776/suppl/GSM4816776%5FPoly01%5Ffeatures%2Etsv%2Egz\n",
    "!cd {kim_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4816nnn/GSM4816776/suppl/GSM4816776%5FPoly01%5Fmatrix%2Emtx%2Egz\n",
    "    \n",
    "!mkdir {kim_dir}/GSM4816776\n",
    "!mv {kim_dir}/GSM4816776_Poly01_barcodes.tsv.gz {kim_dir}/GSM4816776/barcodes.tsv.gz\n",
    "!mv {kim_dir}/GSM4816776_Poly01_features.tsv.gz {kim_dir}/GSM4816776/features.tsv.gz\n",
    "!mv {kim_dir}/GSM4816776_Poly01_matrix.mtx.gz {kim_dir}/GSM4816776/matrix.mtx.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9d15015a-7be6-4526-867a-c3cd10d3c052",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!cd {kim_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4816nnn/GSM4816777/suppl/GSM4816777%5FPoly02%5Fbarcodes%2Etsv%2Egz\n",
    "!cd {kim_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4816nnn/GSM4816777/suppl/GSM4816777%5FPoly02%5Ffeatures%2Etsv%2Egz\n",
    "!cd {kim_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4816nnn/GSM4816777/suppl/GSM4816777%5FPoly02%5Fmatrix%2Emtx%2Egz\n",
    "    \n",
    "!mkdir {kim_dir}/GSM4816777\n",
    "!mv {kim_dir}/GSM4816777_Poly02_barcodes.tsv.gz {kim_dir}/GSM4816777/barcodes.tsv.gz\n",
    "!mv {kim_dir}/GSM4816777_Poly02_features.tsv.gz {kim_dir}/GSM4816777/features.tsv.gz\n",
    "!mv {kim_dir}/GSM4816777_Poly02_matrix.mtx.gz {kim_dir}/GSM4816777/matrix.mtx.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ad53bdc7-cdab-4ca4-91fe-64d812fc3b79",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!cd {kim_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4816nnn/GSM4816778/suppl/GSM4816778%5FPoly03%5Fbarcodes%2Etsv%2Egz\n",
    "!cd {kim_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4816nnn/GSM4816778/suppl/GSM4816778%5FPoly03%5Ffeatures%2Etsv%2Egz\n",
    "!cd {kim_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4816nnn/GSM4816778/suppl/GSM4816778%5FPoly03%5Fmatrix%2Emtx%2Egz\n",
    "    \n",
    "!mkdir {kim_dir}/GSM4816778\n",
    "!mv {kim_dir}/GSM4816778_Poly03_barcodes.tsv.gz {kim_dir}/GSM4816778/barcodes.tsv.gz\n",
    "!mv {kim_dir}/GSM4816778_Poly03_features.tsv.gz {kim_dir}/GSM4816778/features.tsv.gz\n",
    "!mv {kim_dir}/GSM4816778_Poly03_matrix.mtx.gz {kim_dir}/GSM4816778/matrix.mtx.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "356fcd74-e33d-42d3-abbf-4ab18ba42613",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!cd {kim_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4816nnn/GSM4816779/suppl/GSM4816779%5FPoly04%5Fbarcodes%2Etsv%2Egz\n",
    "!cd {kim_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4816nnn/GSM4816779/suppl/GSM4816779%5FPoly04%5Ffeatures%2Etsv%2Egz\n",
    "!cd {kim_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4816nnn/GSM4816779/suppl/GSM4816779%5FPoly04%5Fmatrix%2Emtx%2Egz\n",
    "    \n",
    "!mkdir {kim_dir}/GSM4816779\n",
    "!mv {kim_dir}/GSM4816779_Poly04_barcodes.tsv.gz {kim_dir}/GSM4816779/barcodes.tsv.gz\n",
    "!mv {kim_dir}/GSM4816779_Poly04_features.tsv.gz {kim_dir}/GSM4816779/features.tsv.gz\n",
    "!mv {kim_dir}/GSM4816779_Poly04_matrix.mtx.gz {kim_dir}/GSM4816779/matrix.mtx.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "96fd5d17-b17c-4a7b-8664-2281c52317dc",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_kim_GSM4816776 = sc.read_10x_mtx(data_dir + '/kim_2021/GSM4816776')\n",
    "metadata_assignment(adata_kim_GSM4816776, 'Kim', 2021, '0')\n",
    "\n",
    "adata_kim_GSM4816777 = sc.read_10x_mtx(data_dir + '/kim_2021/GSM4816777')\n",
    "metadata_assignment(adata_kim_GSM4816777, 'Kim', 2021, '1')\n",
    "\n",
    "adata_kim_GSM4816778 = sc.read_10x_mtx(data_dir + '/kim_2021/GSM4816778')\n",
    "metadata_assignment(adata_kim_GSM4816778, 'Kim', 2021, '2')\n",
    "\n",
    "adata_kim_GSM4816779 = sc.read_10x_mtx(data_dir + '/kim_2021/GSM4816779')\n",
    "metadata_assignment(adata_kim_GSM4816779, 'Kim', 2021, '3')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "28af9b56-4b19-482c-ad9a-ad5cc079be3e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_kim = sc.AnnData.concatenate(adata_kim_GSM4816776, adata_kim_GSM4816777, adata_kim_GSM4816778, adata_kim_GSM4816779, \n",
    "                                   batch_key=\"Internal sample identifier\", batch_categories=['0', '1', '2', '3'])\n",
    "adata_kim.write_h5ad(kim_dir + '/kim_2021_ctrl_human.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "72e57b43-2d28-4222-960b-952e3103d9ff",
   "metadata": {},
   "source": [
    "## Liu et al. 2021 (human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fff1d5f6-ff1c-4d02-9059-9d49e7df52f0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "liu_dir = data_dir + '/liu_2021'\n",
    "os.makedirs(liu_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "74623c0b-4e73-4dd3-8204-7309d943fc52",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "csv_liu = pd.read_csv(liu_dir + '/liu_2021.csv', index_col = 0).transpose()\n",
    "metadata_liu = pd.read_csv(liu_dir + '/liu_2021_metadata.csv', index_col = 0)\n",
    "metadata_liu.index = [i.replace('-', '.') for i in metadata_liu.index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e5a9f48b-9a9e-415d-b2b5-bc136439f8ab",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_liu = sc.AnnData(csv_liu)\n",
    "adata_liu.obs = metadata_liu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d52fbc11-5e89-4f34-9198-d2130ffc8b1b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "list_adatas = []\n",
    "\n",
    "list_donors = ['K007CASE', 'K007CTRL', 'K009CASE', 'K009CTRL', 'K013CASE', 'K013CTRL', 'K012CASE', 'K012CTRL']\n",
    "for donor in list_donors:\n",
    "    adata_donor = adata_liu[adata_liu.obs['orig.ident'] == donor].copy()\n",
    "    adata_donor = metadata_assignment(adata_donor, 'Liu', 2021, donor, do_return=True, do_sparse=False)\n",
    "    list_adatas.append(adata_donor)\n",
    "    \n",
    "adata_liu_all = sc.AnnData.concatenate(*list_adatas, batch_key=\"Internal sample identifier\", batch_categories=list_donors)\n",
    "\n",
    "adata_liu_2021_ctrl_human = adata_liu_all[adata_liu_all.obs['Group'] == 'CTRL']\n",
    "adata_liu_2021_keloid = adata_liu_all[adata_liu_all.obs['Group'] == 'CASE']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "39a5efa2-37f1-4dda-a753-8f5d2c041452",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_liu_2021_ctrl_human"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "efe78ab8-d166-4514-9ebe-4d0e3c873889",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_liu_2021_keloid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "21b21212-f606-4269-bebf-c90ee1f7b343",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_liu_2021_keloid.write_h5ad(liu_dir + '/adata_liu_2021_keloid_human.h5')\n",
    "adata_liu_2021_ctrl_human.write_h5ad(liu_dir + '/adata_liu_2021_ctrl_human.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9ad81e6d-e6d5-41a1-b969-bf51a09db905",
   "metadata": {},
   "source": [
    "## Mariottoni et al. 2021 (human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8e90de2e-64d5-4aa2-bc42-be66d6f252fc",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "mariottoni_2021_dir = data_dir + '/mariottoni_2021'\n",
    "os.makedirs(mariottoni_2021_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "547e25f9-a72b-4515-9f20-246ee163426b",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for gsm, name in zip(['GSM5352392', 'GSM5352393', 'GSM5352394', 'GSM5352395'], ['HS_1', 'HS_2', 'HS_3', 'HC']):\n",
    "    !cd {mariottoni_2021_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM5352nnn/{gsm}/suppl/{gsm}%5F{name}%2Ebarcodes%2Etsv%2Egz\n",
    "    !cd {mariottoni_2021_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM5352nnn/{gsm}/suppl/{gsm}%5F{name}%2Efeatures%2Etsv%2Egz\n",
    "    !cd {mariottoni_2021_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM5352nnn/{gsm}/suppl/{gsm}%5F{name}%2Ematrix%2Emtx%2Egz\n",
    "\n",
    "    !mkdir {mariottoni_2021_dir}/{gsm}\n",
    "    !mv {mariottoni_2021_dir}/{gsm}_{name}.barcodes.tsv.gz {mariottoni_2021_dir}/{gsm}/barcodes.tsv.gz\n",
    "    !mv {mariottoni_2021_dir}/{gsm}_{name}.features.tsv.gz {mariottoni_2021_dir}/{gsm}/features.tsv.gz\n",
    "    !mv {mariottoni_2021_dir}/{gsm}_{name}.matrix.mtx.gz {mariottoni_2021_dir}/{gsm}/matrix.mtx.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2c33cc5f-84f5-4f61-9e50-a529588389e6",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "mariottoni_2021_HS_1 = sc.read_10x_mtx(mariottoni_2021_dir + '/GSM5352392')\n",
    "mariottoni_2021_HS_1.var_names_make_unique()\n",
    "metadata_assignment(mariottoni_2021_HS_1, 'Mariottoni', 2021, 'HS_1')\n",
    "\n",
    "mariottoni_2021_HS_2 = sc.read_10x_mtx(mariottoni_2021_dir + '/GSM5352393')\n",
    "mariottoni_2021_HS_2.var_names_make_unique()\n",
    "metadata_assignment(mariottoni_2021_HS_2, 'Mariottoni', 2021, 'HS_2')\n",
    "\n",
    "mariottoni_2021_HS_3 = sc.read_10x_mtx(mariottoni_2021_dir + '/GSM5352394')\n",
    "mariottoni_2021_HS_3.var_names_make_unique()\n",
    "metadata_assignment(mariottoni_2021_HS_3, 'Mariottoni', 2021, 'HS_3')\n",
    "\n",
    "mariottoni_2021_HC = sc.read_10x_mtx(mariottoni_2021_dir + '/GSM5352395')\n",
    "mariottoni_2021_HC.var_names_make_unique()\n",
    "metadata_assignment(mariottoni_2021_HC, 'Mariottoni', 2021, 'HC')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "48735c9b-0ba3-4543-ac06-98311e7b981b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "mariottoni_2021_HS = sc.AnnData.concatenate(mariottoni_2021_HS_1, mariottoni_2021_HS_2, mariottoni_2021_HS_3, \n",
    "                                   batch_key=\"Internal sample identifier\", batch_categories=['HS_1', 'HS_2', 'HS_3'])\n",
    "mariottoni_2021_HS.write_h5ad(mariottoni_2021_dir + '/mariottoni_2021_HS_human.h5')\n",
    "\n",
    "mariottoni_2021_HC.write_h5ad(mariottoni_2021_dir + '/mariottoni_2021_ctrl_human.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e44afda9-8faa-49f8-a18f-a923b5dbc750",
   "metadata": {},
   "source": [
    "## Mirizio et al. 2020 (human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5608d4aa-e41a-4ae4-b0d8-86a39432d1e4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "mirizio_dir = data_dir + '/mirizio_2020'\n",
    "os.makedirs(mirizio_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f782b787-acf6-48f0-83fa-21f2d4385350",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {mirizio_dir} && parallel-fastq-dump -s SRR12955136 --gzip --split-files -t 8 # Cryo P1 222\n",
    "!cd {mirizio_dir} && parallel-fastq-dump -s SRR12955137 --gzip --split-files -t 8\n",
    "!cd {mirizio_dir} && parallel-fastq-dump -s SRR12955138 --gzip --split-files -t 8\n",
    "!cd {mirizio_dir} && parallel-fastq-dump -s SRR12955139 --gzip --split-files -t 8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b100b3c3-5a28-4a6c-8de3-cab426aff604",
   "metadata": {},
   "outputs": [],
   "source": [
    "for SRR_idx, SRR in enumerate(['SRR12955136', 'SRR12955137', 'SRR12955138', 'SRR12955139']):\n",
    "    os.rename(f'{mirizio_dir}/{SRR}_2.fastq.gz', f'{mirizio_dir}/Cryo_P1_L00{SRR_idx + 1}_R1_001.fastq.gz')\n",
    "    os.rename(f'{mirizio_dir}/{SRR}_3.fastq.gz', f'{mirizio_dir}/Cryo_P1_L00{SRR_idx + 1}_R2_001.fastq.gz')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "97f55996-7b89-4822-98c2-4ccf630ddfec",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {mirizio_dir} && parallel-fastq-dump -s SRR12955140 --gzip --split-files -t 8 # RPMI P1 223\n",
    "!cd {mirizio_dir} && parallel-fastq-dump -s SRR12955141 --gzip --split-files -t 8\n",
    "!cd {mirizio_dir} && parallel-fastq-dump -s SRR12955142 --gzip --split-files -t 8\n",
    "!cd {mirizio_dir} && parallel-fastq-dump -s SRR12955143 --gzip --split-files -t 8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "275b4d33-ed15-4283-8a3e-91ac98c1eab1",
   "metadata": {},
   "outputs": [],
   "source": [
    "for SRR_idx, SRR in enumerate(['SRR12955140', 'SRR12955141', 'SRR12955142', 'SRR12955143']):\n",
    "    os.rename(f'{mirizio_dir}/{SRR}_2.fastq.gz', f'{mirizio_dir}/RPMI_P1_L00{SRR_idx + 1}_R1_001.fastq.gz')\n",
    "    os.rename(f'{mirizio_dir}/{SRR}_3.fastq.gz', f'{mirizio_dir}/RPMI_P1_L00{SRR_idx + 1}_R2_001.fastq.gz')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "967da679-64de-4d70-ab49-890a673df41e",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {mirizio_dir} && parallel-fastq-dump -s SRR12955144 --gzip --split-files -t 8 # Cryo P2 267\n",
    "!cd {mirizio_dir} && parallel-fastq-dump -s SRR12955145 --gzip --split-files -t 8\n",
    "!cd {mirizio_dir} && parallel-fastq-dump -s SRR12955146 --gzip --split-files -t 8\n",
    "!cd {mirizio_dir} && parallel-fastq-dump -s SRR12955147 --gzip --split-files -t 8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "85fb5326-bd32-464e-b104-fd15d19f4632",
   "metadata": {},
   "outputs": [],
   "source": [
    "for SRR_idx, SRR in enumerate(['SRR12955144', 'SRR12955145', 'SRR12955146', 'SRR12955147']):\n",
    "    os.rename(f'{mirizio_dir}/{SRR}_2.fastq.gz', f'{mirizio_dir}/untrimmed_Cryo_P2_L00{SRR_idx + 1}_R1_001.fastq.gz')\n",
    "    os.rename(f'{mirizio_dir}/{SRR}_3.fastq.gz', f'{mirizio_dir}/untrimmed_Cryo_P2_L00{SRR_idx + 1}_R2_001.fastq.gz')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "66ae1e2f-5b45-4dc1-8927-1d7731cacd7a",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {mirizio_dir} && parallel-fastq-dump -s SRR12955148 --gzip --split-files -t 8 # RPMI P2 268\n",
    "!cd {mirizio_dir} && parallel-fastq-dump -s SRR12955149 --gzip --split-files -t 8\n",
    "!cd {mirizio_dir} && parallel-fastq-dump -s SRR12955150 --gzip --split-files -t 8\n",
    "!cd {mirizio_dir} && parallel-fastq-dump -s SRR12955151 --gzip --split-files -t 8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "12bd17e0-eb85-4dfa-a8ee-e8db7406b80c",
   "metadata": {},
   "outputs": [],
   "source": [
    "for SRR_idx, SRR in enumerate(['SRR12955148', 'SRR12955149', 'SRR12955150', 'SRR12955151']):\n",
    "    os.rename(f'{mirizio_dir}/{SRR}_2.fastq.gz', f'{mirizio_dir}/untrimmed_RPMI_P2_L00{SRR_idx + 1}_R1_001.fastq.gz')\n",
    "    os.rename(f'{mirizio_dir}/{SRR}_3.fastq.gz', f'{mirizio_dir}/untrimmed_RPMI_P2_L00{SRR_idx + 1}_R2_001.fastq.gz')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e2adf8cf-168c-4992-8315-a44cb843638e",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {mirizio_dir} && parallel-fastq-dump -s SRR12955152 --gzip --split-files -t 8 # Cryo P3 272\n",
    "!cd {mirizio_dir} && parallel-fastq-dump -s SRR12955153 --gzip --split-files -t 8\n",
    "!cd {mirizio_dir} && parallel-fastq-dump -s SRR12955154 --gzip --split-files -t 8\n",
    "!cd {mirizio_dir} && parallel-fastq-dump -s SRR12955155 --gzip --split-files -t 8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2095acce-5939-45ee-bf0d-ebd927200dd1",
   "metadata": {},
   "outputs": [],
   "source": [
    "for SRR_idx, SRR in enumerate(['SRR12955152', 'SRR12955153', 'SRR12955154', 'SRR12955155']):\n",
    "    os.rename(f'{mirizio_dir}/{SRR}_2.fastq.gz', f'{mirizio_dir}/untrimmed_Cryo_P3_L00{SRR_idx + 1}_R1_001.fastq.gz')\n",
    "    os.rename(f'{mirizio_dir}/{SRR}_3.fastq.gz', f'{mirizio_dir}/untrimmed_Cryo_P3_L00{SRR_idx + 1}_R2_001.fastq.gz')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a3cd7ca5-421a-4b83-93a6-253b3579180a",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {mirizio_dir} && parallel-fastq-dump -s SRR12955156 --gzip --split-files -t 8 # RPMI P3 273\n",
    "!cd {mirizio_dir} && parallel-fastq-dump -s SRR12955157 --gzip --split-files -t 8\n",
    "!cd {mirizio_dir} && parallel-fastq-dump -s SRR12955158 --gzip --split-files -t 8\n",
    "!cd {mirizio_dir} && parallel-fastq-dump -s SRR12955159 --gzip --split-files -t 8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "765a46e3-e6ea-4b68-81d8-30a1b94c08ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "for SRR_idx, SRR in enumerate(['SRR12955156', 'SRR12955157', 'SRR12955158', 'SRR12955159']):\n",
    "    os.rename(f'{mirizio_dir}/{SRR}_2.fastq.gz', f'{mirizio_dir}/untrimmed_RPMI_P3_L00{SRR_idx + 1}_R1_001.fastq.gz')\n",
    "    os.rename(f'{mirizio_dir}/{SRR}_3.fastq.gz', f'{mirizio_dir}/untrimmed_RPMI_P3_L00{SRR_idx + 1}_R2_001.fastq.gz')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9edf2680-78f0-4f73-8951-71d56b5e0069",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame({'name': ['Cryo_P1', 'Cryo_P2', 'Cryo_P3',\n",
    "                            'RPMI_P1', 'RPMI_P2', 'RPMI_P3', ], 'technology': ['10xv2'] * 6, \n",
    "                   'targetnumcells': [1000] * 6})\n",
    "df.to_csv(mirizio_dir + '/metadata.tab', sep='\\t', index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "afc655d6-4382-4ab7-be6c-e5d42c9a48ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -rf {mirizio_dir}/*_1.fastq.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b2aad645-80fc-4637-90bb-ea8382693d27",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {kim_dir} && loompy fromfq Kim_2020_HC5.loom Kim_2020_HC5 {human_gencode_dir} metadata.tab \\\n",
    "Kim_2020_HC5_L001_R1_001.fastq.gz Kim_2020_HC5_L001_R2_001.fastq.gz "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5d69e908-4b8d-4008-b87d-418219dc961f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# We will trim the bases to get the first 26 bp in R1 and the first 98 in R2\n",
    "for name in ['Cryo_P2', 'Cryo_P3', 'RPMI_P2', 'RPMI_P3']:\n",
    "    for lane in ['1', '2', '3', '4']:\n",
    "        os.system(f'seqtk trimfq -e 124 {mirizio_dir}/untrimmed_{name}_L00{lane}_R1_001.fastq.gz > {mirizio_dir}/{name}_L00{lane}_R1_001.fastq.gz')\n",
    "        os.system(f'seqtk trimfq -e 52  {mirizio_dir}/untrimmed_{name}_L00{lane}_R2_001.fastq.gz > {mirizio_dir}/{name}_L00{lane}_R2_001.fastq.gz')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c94bb1d0-dad0-4beb-aa63-f31af39e3e8f",
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -rf {mirizio_dir}/untrimmed_*.fastq.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "051cb328-85ce-49be-b411-50cde33cee3d",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {mirizio_dir} && loompy fromfq Cryo_P1.loom Cryo_P1 {human_gencode_dir} metadata.tab \\\n",
    "Cryo_P1_L001_R1_001.fastq.gz Cryo_P1_L001_R2_001.fastq.gz Cryo_P1_L002_R1_001.fastq.gz Cryo_P1_L002_R2_001.fastq.gz Cryo_P1_L003_R1_001.fastq.gz Cryo_P1_L003_R2_001.fastq.gz Cryo_P1_L003_R1_001.fastq.gz Cryo_P1_L003_R2_001.fastq.gz "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4e091622-c67a-4bd8-ac3c-3a0056973d33",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {mirizio_dir} && loompy fromfq Cryo_P2.loom Cryo_P2 {human_gencode_dir} metadata.tab \\\n",
    "Cryo_P2_L001_R1_001.fastq.gz Cryo_P2_L001_R2_001.fastq.gz Cryo_P2_L002_R1_001.fastq.gz Cryo_P2_L002_R2_001.fastq.gz Cryo_P2_L003_R1_001.fastq.gz Cryo_P2_L003_R2_001.fastq.gz Cryo_P2_L003_R1_001.fastq.gz Cryo_P2_L003_R2_001.fastq.gz "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "63f3c489-4ef9-41e0-b1d4-d4eabcd49fe0",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {mirizio_dir} && loompy fromfq Cryo_P3.loom Cryo_P3 {human_gencode_dir} metadata.tab \\\n",
    "Cryo_P3_L001_R1_001.fastq.gz Cryo_P3_L001_R2_001.fastq.gz Cryo_P3_L002_R1_001.fastq.gz Cryo_P3_L002_R2_001.fastq.gz Cryo_P3_L003_R1_001.fastq.gz Cryo_P3_L003_R2_001.fastq.gz Cryo_P3_L003_R1_001.fastq.gz Cryo_P3_L003_R2_001.fastq.gz "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "39b5f748-809c-4b3e-a79d-03544626b98d",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {mirizio_dir} && loompy fromfq RPMI_P1.loom RPMI_P1 {human_gencode_dir} metadata.tab \\\n",
    "RPMI_P1_L001_R1_001.fastq.gz RPMI_P1_L001_R2_001.fastq.gz RPMI_P1_L002_R1_001.fastq.gz RPMI_P1_L002_R2_001.fastq.gz RPMI_P1_L003_R1_001.fastq.gz RPMI_P1_L003_R2_001.fastq.gz RPMI_P1_L003_R1_001.fastq.gz RPMI_P1_L003_R2_001.fastq.gz "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c7cb0695-bf07-4ac7-b183-3889fa5866f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {mirizio_dir} && loompy fromfq RPMI_P2.loom RPMI_P2 {human_gencode_dir} metadata.tab \\\n",
    "RPMI_P2_L001_R1_001.fastq.gz RPMI_P2_L001_R2_001.fastq.gz RPMI_P2_L002_R1_001.fastq.gz RPMI_P2_L002_R2_001.fastq.gz RPMI_P2_L003_R1_001.fastq.gz RPMI_P2_L003_R2_001.fastq.gz RPMI_P2_L003_R1_001.fastq.gz RPMI_P2_L003_R2_001.fastq.gz "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "93aef74b-8c10-45a2-a2a0-b1fd19f4105c",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {mirizio_dir} && loompy fromfq RPMI_P3.loom RPMI_P3 {human_gencode_dir} metadata.tab \\\n",
    "RPMI_P3_L001_R1_001.fastq.gz RPMI_P3_L001_R2_001.fastq.gz RPMI_P3_L002_R1_001.fastq.gz RPMI_P3_L002_R2_001.fastq.gz RPMI_P3_L003_R1_001.fastq.gz RPMI_P3_L003_R2_001.fastq.gz RPMI_P3_L003_R1_001.fastq.gz RPMI_P3_L003_R2_001.fastq.gz "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ec035b13-196d-46ac-8d3c-56f246c98d3e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_mirizio_Cryo_P1 = sc.read_loom(mirizio_dir + '/Cryo_P1.loom')\n",
    "adata_mirizio_Cryo_P1.var_names_make_unique()\n",
    "metadata_assignment(adata_mirizio_Cryo_P1, 'Mirizio', 2020, 'Cryo_P1')\n",
    "\n",
    "adata_mirizio_Cryo_P2 = sc.read_loom(mirizio_dir + '/Cryo_P2.loom')\n",
    "adata_mirizio_Cryo_P2.var_names_make_unique()\n",
    "metadata_assignment(adata_mirizio_Cryo_P2, 'Mirizio', 2020, 'Cryo_P2')\n",
    "\n",
    "adata_mirizio_Cryo_P3 = sc.read_loom(mirizio_dir + '/Cryo_P3.loom')\n",
    "adata_mirizio_Cryo_P3.var_names_make_unique()\n",
    "metadata_assignment(adata_mirizio_Cryo_P3, 'Mirizio', 2020, 'Cryo_P3')\n",
    "\n",
    "adata_mirizio_RPMI_P1 = sc.read_loom(mirizio_dir + '/RPMI_P1.loom')\n",
    "adata_mirizio_RPMI_P1.var_names_make_unique()\n",
    "metadata_assignment(adata_mirizio_RPMI_P1, 'Mirizio', 2020, 'RPMI_1')\n",
    "\n",
    "adata_mirizio_RPMI_P2 = sc.read_loom(mirizio_dir + '/RPMI_P2.loom')\n",
    "adata_mirizio_RPMI_P2.var_names_make_unique()\n",
    "metadata_assignment(adata_mirizio_RPMI_P2, 'Mirizio', 2020, 'RPMI_2')\n",
    "\n",
    "adata_mirizio_RPMI_P3 = sc.read_loom(mirizio_dir + '/RPMI_P3.loom')\n",
    "adata_mirizio_RPMI_P3.var_names_make_unique()\n",
    "metadata_assignment(adata_mirizio_RPMI_P3, 'Mirizio', 2020, 'RPMI_3')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bda86caf-c680-43fe-9a55-1ec30e49f79c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_mirizio = sc.AnnData.concatenate(adata_mirizio_Cryo_P1, adata_mirizio_Cryo_P2, adata_mirizio_Cryo_P3, \n",
    "                                   adata_mirizio_RPMI_P1, adata_mirizio_RPMI_P2, adata_mirizio_RPMI_P3, \n",
    "                                   batch_categories=['Cryo_P1', 'Cryo_P2', 'Cryo_P3', \n",
    "                                                     'RPMI_1', 'RPMI_2', 'RPMI_3'], batch_key=\"Internal sample identifier\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "386a4b1d-e2d9-473f-9ec7-2db6d0ab2657",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_mirizio.write_h5ad(mirizio_dir + '/adata_mirizio_2020_scleroderma_human.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "explicit-breakdown",
   "metadata": {},
   "source": [
    "## Philippeos et al. 2018 (human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "activated-appointment",
   "metadata": {},
   "outputs": [],
   "source": [
    "phil_dir = data_dir + '/Philippeos_2018'\n",
    "os.makedirs(phil_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "divine-spectacular",
   "metadata": {},
   "outputs": [],
   "source": [
    "!wget -P {phil_dir} https://ftp.ncbi.nlm.nih.gov/geo/series/GSE109nnn/GSE109822/suppl/GSE109822%5FCD3145%2Ecsv%2Egz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "adapted-reference",
   "metadata": {},
   "outputs": [],
   "source": [
    "!wget -P {phil_dir} https://ftp.ncbi.nlm.nih.gov/geo/series/GSE109nnn/GSE109822/suppl/GSE109822%5FCD90%2Ecsv%2Egz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "included-debut",
   "metadata": {},
   "outputs": [],
   "source": [
    "!gunzip {phil_dir}/*.gz -f"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d8ff333d-f96c-40f4-b0aa-867da45fa24b",
   "metadata": {},
   "source": [
    "## Popescu et al. 2019 (human)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "db25a15f-98c6-43fa-bf2b-7619770c53a6",
   "metadata": {},
   "source": [
    "### Direct h5ad download"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "195608dd-ac07-4094-b94c-2d2370a128da",
   "metadata": {},
   "outputs": [],
   "source": [
    "!aria2c -x 16 https://zenodo.org/record/4536165/files/fetal_submission.h5ad?download=1 -d {popescu_dir} -o popescu_2019.h5ad"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7574348d-9212-4419-b61f-7cefe50932dd",
   "metadata": {},
   "source": [
    "### FASTQ processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "134b0117-27ae-4aa3-8fc4-39b062cff1e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "!aria2c -x 16 https://www.ebi.ac.uk/arrayexpress/files/E-MTAB-7407/E-MTAB-7407.sdrf.txt -d {popescu_dir} -o acctable.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "686f2a28-0ce8-4aba-8141-e9743c5df7e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "popescu_metadata = pd.read_csv(popescu_dir + '/acctable.txt', sep='\\t')\n",
    "popescu_metadata_skin = popescu_metadata[popescu_metadata['Characteristics[organism part]'] == 'skin'].reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a4667eef-d818-4d3b-894d-9521d5e1dc64",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame({'name': [f\"{i}_{popescu_metadata_skin['Source Name'].values[i]}_{popescu_metadata_skin['Characteristics[individual]'].values[i]}_{popescu_metadata_skin['Characteristics[facs sorting]'].values[i].replace('/', '-').replace(' ', '-')}\" for i in range(len(popescu_metadata_skin))], \n",
    "                   'technology': ['10xv2'] * len(popescu_metadata_skin), \n",
    "                   'targetnumcells': [1000] * len(popescu_metadata_skin)})\n",
    "\n",
    "df.to_csv(popescu_dir + '/metadata.tab', sep='\\t', index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1bcf423b-bd59-4356-983c-1811ae2e4c27",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd  {popescu_dir} && cat metadata.tab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "391d8074-f660-4733-b30a-d2519cfd1593",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(popescu_metadata_skin))\n",
    "for idx, name, indv, facs, f1, f2 in tqdm(zip(range(len(popescu_metadata_skin)), \n",
    "                                                    popescu_metadata_skin['Source Name'].values, \n",
    "                                    popescu_metadata_skin['Characteristics[individual]'].values, \n",
    "                                    popescu_metadata_skin['Characteristics[facs sorting]'].values,\n",
    "                                    popescu_metadata_skin['Comment[FASTQ_URI]'].values, \n",
    "                                    popescu_metadata_skin['Comment[FASTQ_URI].1'].values)):\n",
    "    \n",
    "    facs = facs.replace('/', '-').replace(' ', '-')\n",
    "    str_file = f'{idx}_{name}_{indv}_{facs}'\n",
    "    \n",
    "    if os.path.exists(f'{popescu_dir}/Popescu_2019_{str_file}.loom'):\n",
    "        print(f'Popescu_2019_{str_file}.loom EXISTS!')\n",
    "    else:\n",
    "        os.system(f'cd {popescu_dir} && aria2c -x 16 --file-allocation=none {f1} -d {popescu_dir} -o {str_file}_S1_L001_R1_001.fastq.gz')\n",
    "        os.system(f'cd {popescu_dir} && aria2c -x 16 --file-allocation=none {f2} -d {popescu_dir} -o {str_file}_S1_L001_R2_001.fastq.gz')\n",
    "\n",
    "        os.system(f'cd {popescu_dir} && loompy fromfq Popescu_2019_{str_file}.loom {str_file} {human_gencode_dir} metadata.tab {str_file}_S1_L001_R1_001.fastq.gz {str_file}_S1_L001_R2_001.fastq.gz ')\n",
    "\n",
    "        os.system(f'rm {popescu_dir}/{str_file}_S1_L001_R1_001.fastq.gz')\n",
    "        os.system(f'rm {popescu_dir}/{str_file}_S1_L001_R2_001.fastq.gz')\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "efa3f97b-c0f6-40f4-9f21-85487e8eeacb",
   "metadata": {},
   "source": [
    "## Reynolds et al. 2021 (human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "709ff1fc-b3bd-4e13-b523-ee2dfc04da8a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Refer to https://github.com/alexmascension/revisit_reynolds_fb to see to main processing files."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "255f51e8-bd2d-45ca-b5cb-8ab4975770a4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "reynolds_dir = data_dir + '/reynolds_2021'\n",
    "\n",
    "reynolds_2021_ctrl_human_fb = sc.read(reynolds_dir + '/adata_reynolds_healthy_fb.h5ad', \n",
    "                                    backup_url='https://zenodo.org/record/4708700/files/adata_reynolds_healthy_fb.h5ad?download=1')\n",
    "\n",
    "del reynolds_2021_ctrl_human_fb.uns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0ea73908-3c05-485f-a0e3-82e557854b06",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "reynolds_2021_ctrl_human_fb.obs['Internal sample identifier'] = reynolds_2021_ctrl_human_fb.obs['sample_id']\n",
    "\n",
    "list_adatas_control = []\n",
    "\n",
    "for cat in reynolds_2021_ctrl_human_fb.obs['Internal sample identifier'].cat.categories:\n",
    "    adata = reynolds_2021_ctrl_human_fb[reynolds_2021_ctrl_human_fb.obs['Internal sample identifier'] == cat]\n",
    "    adata = metadata_assignment(adata, 'Reynolds', 2021, cat, do_return=True)\n",
    "    list_adatas_control.append(adata)\n",
    "\n",
    "reynolds_2021_ctrl_human_fb = sc.AnnData.concatenate(*list_adatas_control, batch_categories=reynolds_2021_ctrl_human_fb.obs['Internal sample identifier'].cat.categories, batch_key=\"Internal sample identifier\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fcc1c056-c542-434b-8420-d5deb4e91849",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "reynolds_2021_ctrl_human_fb.write_h5ad(reynolds_dir + '/reynolds_2021_ctrl_human_fb.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "79b583d4-40d8-4057-a7a7-b670646d1d95",
   "metadata": {},
   "source": [
    "## Rindler et al. 2021 (human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e20a7cfe-5062-41cb-84ba-6e0428dbdf3c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "rindler_2021_dir = data_dir + '/rindler_2021'\n",
    "os.makedirs(rindler_2021_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f8ee021b-bc37-4db1-82eb-a24d4394d958",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for gsm, name in zip(['GSM5534590', 'GSM5534591', 'GSM5534592', 'GSM5534593'], ['P112 HC', 'P115 HC', 'P116 HC', 'P121 HC']):\n",
    "    !cd {rindler_2021_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM5534nnn/{gsm}/suppl/{gsm}%5F{name.replace(' ', '_')}%5Fbarcodes%2Etsv%2Egz\n",
    "    !cd {rindler_2021_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM5534nnn/{gsm}/suppl/{gsm}%5F{name.replace(' ', '_')}%5Ffeatures%2Etsv%2Egz\n",
    "    !cd {rindler_2021_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM5534nnn/{gsm}/suppl/{gsm}%5F{name.replace(' ', '_')}%5Fmatrix%2Emtx%2Egz\n",
    "\n",
    "    !mkdir {rindler_2021_dir}/{gsm}\n",
    "    !mv {rindler_2021_dir}/{gsm}_{name.replace(' ', '_')}_barcodes.tsv.gz {rindler_2021_dir}/{gsm}/barcodes.tsv.gz\n",
    "    !mv {rindler_2021_dir}/{gsm}_{name.replace(' ', '_')}_features.tsv.gz {rindler_2021_dir}/{gsm}/features.tsv.gz\n",
    "    !mv {rindler_2021_dir}/{gsm}_{name.replace(' ', '_')}_matrix.mtx.gz {rindler_2021_dir}/{gsm}/matrix.mtx.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d760f2cc-3b77-4100-9a10-ad0377a3678d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "list_adatas = []\n",
    "\n",
    "for gsm, name in zip(['GSM5534590', 'GSM5534591', 'GSM5534592', 'GSM5534593'], ['P112 HC', 'P115 HC', 'P116 HC', 'P121 HC']):\n",
    "    adata = sc.read_10x_mtx(rindler_2021_dir + f'/{gsm}')\n",
    "    adata.var_names_make_unique()\n",
    "    metadata_assignment(adata, 'Rindler', 2021, name)\n",
    "    list_adatas.append(adata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d2cfaa30-5bfe-473d-b4d0-0eb3eff16357",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "rindler_2021_ctrl_human = sc.AnnData.concatenate(*list_adatas, \n",
    "                                   batch_key=\"Internal sample identifier\", batch_categories=['P112 HC', 'P115 HC', 'P116 HC', 'P121 HC'])\n",
    "rindler_2021_ctrl_human.write_h5ad(rindler_2021_dir + '/rindler_2021_ctrl_human.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8770844d-bfed-4aad-b943-9693ec2dabe0",
   "metadata": {},
   "source": [
    "## Saluzzo et al. 2022 (human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "701fc8f8-aaf2-4bad-bf9c-72005c59c3e0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "saluzzo_2022_dir = data_dir + '/saluzzo_2022'\n",
    "os.makedirs(saluzzo_2022_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "19dffc40-c18e-434d-ba5c-3c232c5c664e",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for gsm, name in zip(['GSM5584058', 'GSM5584059', 'GSM5584060'], ['1', '2', '4']):\n",
    "    !cd {saluzzo_2022_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM5584nnn/{gsm}/suppl/{gsm}%5FSSHIV35%5F{name}%5Ftranscriptome%2Etar%2Egz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3ad8244a-d09f-49ea-9a43-24a9bf268d63",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!cd {saluzzo_2022_dir} && cat *.tar.gz | tar zxvf - -i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "432c4814-0601-4caf-849b-24d0e8dac530",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "list_adatas = []\n",
    "\n",
    "for gsm, name in zip(['SSHIV35_1_transcriptome', 'SSHIV35_2_transcriptome', 'SSHIV35_4_transcriptome'], ['H1', 'H2', 'H3']):\n",
    "    adata = sc.read_10x_mtx(saluzzo_2022_dir + f'/{gsm}/filtered_feature_bc_matrix')\n",
    "    adata.var_names_make_unique()\n",
    "    metadata_assignment(adata, 'Saluzzo', 2022, name)\n",
    "    list_adatas.append(adata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6189c5fa-60d4-4e8f-b97d-5cdabaac9372",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "saluzzo_2022_ctrl_human = sc.AnnData.concatenate(*list_adatas, \n",
    "                                   batch_key=\"Internal sample identifier\", batch_categories=['H1', 'H2', 'H3'])\n",
    "saluzzo_2022_ctrl_human.write_h5ad(saluzzo_2022_dir + '/saluzzo_2022_ctrl_human.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cross-course",
   "metadata": {},
   "source": [
    "## Solé-Boldo et al. 2020 (human)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d713349c-b6cb-4e99-8fc9-349370456041",
   "metadata": {},
   "source": [
    "### Young samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "olive-aluminum",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sole_dir = data_dir + '/Sole-Boldo_2020'\n",
    "os.makedirs(sole_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "racial-biology",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!cd {sole_dir} && fastq-dump SRR9036396 --gzip --split-files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "greater-wisconsin",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {sole_dir} && fastq-dump SRR9036397 --gzip --split-files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "hidden-exclusive",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame({'name': ['SB2020_y1', 'SB2020_y2'], 'technology': ['10xv2'] * 2, 'targetnumcells': [1000] * 2})\n",
    "df.to_csv(sole_dir + '/metadata.tab', sep='\\t', index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "pressing-powell",
   "metadata": {},
   "outputs": [],
   "source": [
    "!mv {sole_dir}/SRR9036396_1.fastq.gz {sole_dir}/SB2020_y1_L001_R1_001.fastq.gz \n",
    "!mv {sole_dir}/SRR9036396_2.fastq.gz {sole_dir}/SB2020_y1_L001_R2_001.fastq.gz \n",
    "!mv {sole_dir}/SRR9036397_1.fastq.gz {sole_dir}/SB2020_y2_L002_R1_001.fastq.gz \n",
    "!mv {sole_dir}/SRR9036397_2.fastq.gz {sole_dir}/SB2020_y2_L002_R2_001.fastq.gz "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "czech-directory",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {sole_dir} && loompy fromfq SB2020_y1.loom SB2020_y1 {human_gencode_dir} metadata.tab SB2020_y1_L001_R1_001.fastq.gz SB2020_y1_L001_R2_001.fastq.gz \n",
    "\n",
    "!cd {sole_dir} && loompy fromfq SB2020_y2.loom SB2020_y2 {human_gencode_dir} metadata.tab SB2020_y2_L002_R1_001.fastq.gz SB2020_y2_L002_R2_001.fastq.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1f484421-6c18-44f3-961c-a46769021008",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_sole_young_1 = sc.read_loom(sole_dir + '/SB2020_y1.loom')\n",
    "adata_sole_young_1.var_names_make_unique()\n",
    "metadata_assignment(adata_sole_young_1, 'Solé-Boldo', 2020, 'y1')\n",
    "\n",
    "adata_sole_young_2 = sc.read_loom(sole_dir + '/SB2020_y2.loom')\n",
    "adata_sole_young_2.var_names_make_unique()\n",
    "metadata_assignment(adata_sole_young_2, 'Solé-Boldo', 2020, 'y2')\n",
    "\n",
    "adata_sole_young = sc.AnnData.concatenate(adata_sole_young_1, adata_sole_young_2, batch_key=\"Internal sample identifier\", batch_categories=['y1', 'y2'])\n",
    "adata_sole_young.write_h5ad(sole_dir + '/adata_sole_2020_young_human.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "32860e84-d75e-47a0-8f15-39bd6ab43700",
   "metadata": {},
   "source": [
    "### Old samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee74805c-50df-4d89-a34a-dd260ad3026e",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {sole_dir} && fastq-dump SRR9036398 --gzip --split-files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "caca9e9f-606a-47ab-8a00-cae2f1f52bb2",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {sole_dir} && fastq-dump SRR9036399 --gzip --split-files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe538473-85da-443d-9b03-5c41cec5c870",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {sole_dir} && fastq-dump SRR9036400 --gzip --split-files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6eae1a7f-6c5c-4c50-b77c-7643797bd5d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame({'name': ['SB2020_o1', 'SB2020_o2', 'SB2020_o3'], 'technology': ['10xv2'] * 3, 'targetnumcells': [1000] * 3})\n",
    "df.to_csv(sole_dir + '/metadata.tab', sep='\\t', index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b50718e3-6f43-4c18-b439-226cf73530b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "!mv {sole_dir}/SRR9036398_1.fastq.gz {sole_dir}/SB2020_o1_L001_R1_001.fastq.gz \n",
    "!mv {sole_dir}/SRR9036398_2.fastq.gz {sole_dir}/SB2020_o1_L001_R2_001.fastq.gz \n",
    "!mv {sole_dir}/SRR9036399_1.fastq.gz {sole_dir}/SB2020_o2_L002_R1_001.fastq.gz \n",
    "!mv {sole_dir}/SRR9036399_2.fastq.gz {sole_dir}/SB2020_o2_L002_R2_001.fastq.gz \n",
    "!mv {sole_dir}/SRR9036400_1.fastq.gz {sole_dir}/SB2020_o3_L003_R1_001.fastq.gz \n",
    "!mv {sole_dir}/SRR9036400_2.fastq.gz {sole_dir}/SB2020_o3_L003_R2_001.fastq.gz "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "346a2a35-3940-4678-8840-f703f887e3b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {sole_dir} && loompy fromfq SB2020_o1.loom SB2020_o1 {human_gencode_dir} metadata.tab SB2020_o1_L001_R1_001.fastq.gz SB2020_o1_L001_R2_001.fastq.gz\n",
    "\n",
    "!cd {sole_dir} && loompy fromfq SB2020_o2.loom SB2020_o2 {human_gencode_dir} metadata.tab SB2020_o2_L002_R1_001.fastq.gz SB2020_o2_L002_R2_001.fastq.gz\n",
    "\n",
    "!cd {sole_dir} && loompy fromfq SB2020_o3.loom SB2020_o3 {human_gencode_dir} metadata.tab SB2020_o3_L003_R1_001.fastq.gz SB2020_o3_L003_R2_001.fastq.gz "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5c18bafa-6ec7-493b-9c07-d981e8b99c44",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_sole_old_1 = sc.read_loom(sole_dir + '/SB2020_o1.loom')\n",
    "adata_sole_old_1.var_names_make_unique()\n",
    "metadata_assignment(adata_sole_old_1, 'Solé-Boldo', 2020, 'o1')\n",
    "\n",
    "adata_sole_old_2 = sc.read_loom(sole_dir + '/SB2020_o2.loom')\n",
    "adata_sole_old_2.var_names_make_unique()\n",
    "metadata_assignment(adata_sole_old_2, 'Solé-Boldo', 2020, 'o2')\n",
    "\n",
    "adata_sole_old_3 = sc.read_loom(sole_dir + '/SB2020_o3.loom')\n",
    "adata_sole_old_3.var_names_make_unique()\n",
    "metadata_assignment(adata_sole_old_3, 'Solé-Boldo', 2020, 'o3')\n",
    "\n",
    "adata_sole_old = sc.AnnData.concatenate(adata_sole_old_1, adata_sole_old_2, adata_sole_old_3, batch_key=\"Internal sample identifier\", batch_categories=['o1', 'o2', 'o3'])\n",
    "adata_sole_old.write_h5ad(sole_dir + '/adata_sole_2020_old_human.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dd8db5b1-088d-4783-baa4-fd795653e8cf",
   "metadata": {},
   "source": [
    "## Tabib et al. 2018 (human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1ba85b50-541c-4126-823c-cd3ca5576c51",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "tabib_2018_dir = data_dir + '/Tabib_2018'\n",
    "os.makedirs(tabib_2018_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dded2093-d106-4ba6-a60e-616cfd14754d",
   "metadata": {},
   "outputs": [],
   "source": [
    "!wget -P {tabib_dir} https://dom.pitt.edu/wp-content/uploads/2018/10/Skin_6Control_rawUMI.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3b248aa9-c79d-44d6-9856-cdb7b1334652",
   "metadata": {},
   "outputs": [],
   "source": [
    "!wget -P {tabib_dir} https://dom.pitt.edu/wp-content/uploads/2018/10/Skin_6Control_Metadata.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1533a4f9-bb21-4e42-91b8-7e496d03dc04",
   "metadata": {},
   "outputs": [],
   "source": [
    "!unzip -o {tabib_dir}/Skin_6Control_rawUMI.zip -d {tabib_dir}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9c0c9064-d570-46bb-8d45-9e6d9b91282b",
   "metadata": {},
   "outputs": [],
   "source": [
    "!unzip -o {tabib_dir}/Skin_6Control_Metadata.zip -d {tabib_dir}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9b6f1b77-b139-427a-adfb-36e18535ba8c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "tabib_2018_ctrl_human = sc.read_csv(tabib_2018_dir + '/Skin_6Control_rawUMI.csv')\n",
    "tabib_2018_ctrl_human = tabib_2018_ctrl_human.transpose()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0ab57fda-7b59-4e38-af91-f698345856f0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "tabib_2018_ctrl_human.obs['Internal sample identifier'] = [i.split('_')[0] for i in tabib_2018_ctrl_human.obs_names]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a7237ee5-7dc3-428f-a2eb-1a619af3d021",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "tabib_2018_ctrl_human.obs['Internal sample identifier']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "80287d0c-f41b-4917-af7f-2160b119416f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "list_adatas = [ ]\n",
    "\n",
    "list_donors = ['SC1control', 'SC4control', 'SC18control', 'SC32control', 'SC33control', 'SC34control']\n",
    "for donor in list_donors:\n",
    "    adata_donor = tabib_2018_ctrl_human[tabib_2018_ctrl_human.obs['Internal sample identifier'] == donor]\n",
    "    adata_donor = metadata_assignment(adata_donor, 'Tabib', 2018, donor, do_return=True)\n",
    "    list_adatas.append(adata_donor)\n",
    "    \n",
    "tabib_2018_ctrl_human = sc.AnnData.concatenate(*list_adatas, batch_key=\"Internal sample identifier\", batch_categories=list_donors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "11f6d2b4-ee20-41c3-8a62-e447012fa5a8",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "tabib_2018_ctrl_human.obs_names = [i.split('-')[0] for i in tabib_2018_ctrl_human.obs_names]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a22e67e7-d53d-42a2-b2aa-bafff303093c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "tabib_2018_ctrl_human.write_h5ad(tabib_2018_dir + '/adata_tabib_2018_ctrl_human.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4042f803-3f72-4fca-a9e4-665120c2f674",
   "metadata": {},
   "source": [
    "## Tabib et al. 2021 (human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9cd021f-8726-4782-a969-9e2309124977",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "tabib_2021_dir = data_dir + '/Tabib_2021'\n",
    "os.makedirs(tabib_2021_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "184a3492-082f-48cc-a051-121bce8e352c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "GSM_list_control = [('SC1', 'CONTROL', 'GSM4115868'), ('SC4', 'CONTROL', 'GSM4115870'), ('SC18', 'CONTROL', 'GSM4115872'), ('SC32', 'CONTROL', 'GSM4115874'),\n",
    "                    ('SC33', 'CONTROL', 'GSM4115875'), ('SC34', 'CONTROL', 'GSM4115876'), ('SC50', 'CONTROL', 'GSM4115878'),\n",
    "                    ('SC68', 'CONTROL', 'GSM4115880'), ('SC124', 'CONTROL', 'GSM4115885'), ('SC125', 'CONTROL', 'GSM4115886')]\n",
    "GSM_list_SSC = [('SC2', 'SSC', 'GSM4115869'), ('SC5', 'SSC', 'GSM4115871'), ('SC19', 'SSC', 'GSM4115873'), ('SC49', 'SSC', 'GSM4115877'),\n",
    "                ('SC60', 'SSC', 'GSM4115879'), ('SC69', 'SSC', 'GSM4115881'), ('SC70', 'SSC', 'GSM4115882'), ('SC86', 'SSC', 'GSM4115883'),\n",
    "                ('SC119', 'SSC', 'GSM4115884'), ('SC185', 'SSC', 'GSM4115887'), ('SC188', 'SSC', 'GSM4115888'), ('SC189', 'SSC', 'GSM4115889'),]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f3cded56-d78c-4705-8060-f93b2c1f5ccf",
   "metadata": {},
   "outputs": [],
   "source": [
    "for name, _, gsm in GSM_list_control + GSM_list_SSC:\n",
    "    os.system(f\"wget -P {tabib_2021_dir}  ftp://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4115nnn/{gsm}/suppl/{gsm}%5F{name}raw%5Ffeature%5Fbc%5Fmatrix%2Eh5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db762212-da09-4b33-a428-2d75dc35d48e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "list_adatas_control = []\n",
    "\n",
    "for name, _, gsm in GSM_list_control:\n",
    "    adata = sc.read_10x_h5(f\"{tabib_2021_dir}/{gsm}_{name}raw_feature_bc_matrix.h5\")\n",
    "    adata.var_names_make_unique()\n",
    "    sc.pp.filter_cells(adata, min_counts=150)\n",
    "    adata = metadata_assignment(adata, 'Tabib', 2021, name, do_return=True)\n",
    "    list_adatas_control.append(adata)\n",
    "\n",
    "adata_tabib_2021_ctrl_human = sc.AnnData.concatenate(*list_adatas_control, batch_categories=[i[0] for i in GSM_list_control], batch_key=\"Internal sample identifier\")\n",
    "sc.pp.filter_genes(adata_tabib_2021_ctrl_human, min_counts=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "84924a6a-34ff-4e2f-8616-f20f10a11318",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "list_adatas_ssc = []\n",
    "\n",
    "for name, _, gsm in GSM_list_SSC:\n",
    "    adata = sc.read_10x_h5(f\"{tabib_2021_dir}/{gsm}_{name}raw_feature_bc_matrix.h5\")\n",
    "    adata.var_names_make_unique()\n",
    "    sc.pp.filter_cells(adata, min_counts=150)\n",
    "    adata = metadata_assignment(adata, 'Tabib', 2021, name, do_return=True)\n",
    "    list_adatas_ssc.append(adata)\n",
    "\n",
    "adata_tabib_2021_ssc = sc.AnnData.concatenate(*list_adatas_ssc, batch_categories=[i[0] for i in GSM_list_SSC], batch_key=\"Internal sample identifier\")\n",
    "sc.pp.filter_genes(adata_tabib_2021_ssc, min_counts=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8f7d8707-6243-4b7e-b550-51c640630cf6",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_tabib_2021_ctrl_human.write_h5ad(tabib_2021_dir + '/adata_tabib_2021_ctrl_human.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "315aedf9-183a-48a6-ba5a-ffdbc868c8da",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_tabib_2021_ssc.write_h5ad(tabib_2021_dir + '/adata_tabib_2021_ssc_human.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bd3432ac-1dc1-4f2d-bbc0-d0e4a599ac26",
   "metadata": {},
   "source": [
    "## Tabula Sapiens Consortium 2021 (human)\n",
    "\n",
    "### Fastq\n",
    "To access these fastq files you must ask for permission to CZ tabula sapiens (tabula-sapiens@czbiohub.org) and ask for data request.\n",
    "\n",
    "**Downloading their h5ad leads to the same shitty quality dataset!!**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c35b1262-f5aa-4620-8053-63fb171f69bb",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "tsc_dir = data_dir + '/Tabula_Sapiens_Consortium_2021'\n",
    "os.makedirs(tsc_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5ff20698-a5ba-4616-85f1-6247c66ca36c",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame({'name': ['TSP10_Skin_NA_10X_1_1_S5', 'TSP10_Skin_NA_10X_1_2_S6', 'TSP14_Skin_Chest_10X_1_1_S18', 'TSP14_Skin_Abdomen_10X_1_1_S17'], \n",
    "                   'technology': ['10xv3'] * 4, \n",
    "                   'targetnumcells': [1000] * 4})\n",
    "df.to_csv(tsc_dir + '/metadata.tab', sep='\\t', index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "813e3369-36a0-4548-ac4a-fdc11a3baf63",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {tsc_dir} && loompy fromfq TSP10_S5.loom TSP10_Skin_NA_10X_1_1_S5 {human_gencode_dir} metadata.tab \\\n",
    "Pilot_10/TSP10_Skin_NA_10X_1_1/TSP10_Skin_NA_10X_1_1_S5_L001_R1_001.fastq.gz Pilot_10/TSP10_Skin_NA_10X_1_1/TSP10_Skin_NA_10X_1_1_S5_L001_R2_001.fastq.gz \\\n",
    "Pilot_10/TSP10_Skin_NA_10X_1_1/TSP10_Skin_NA_10X_1_1_S5_L002_R1_001.fastq.gz Pilot_10/TSP10_Skin_NA_10X_1_1/TSP10_Skin_NA_10X_1_1_S5_L002_R2_001.fastq.gz \\\n",
    "Pilot_10/TSP10_Skin_NA_10X_1_1/TSP10_Skin_NA_10X_1_1_S5_L003_R1_001.fastq.gz Pilot_10/TSP10_Skin_NA_10X_1_1/TSP10_Skin_NA_10X_1_1_S5_L003_R2_001.fastq.gz \\\n",
    "Pilot_10/TSP10_Skin_NA_10X_1_1/TSP10_Skin_NA_10X_1_1_S5_L004_R1_001.fastq.gz Pilot_10/TSP10_Skin_NA_10X_1_1/TSP10_Skin_NA_10X_1_1_S5_L004_R2_001.fastq.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "be65c5bb-a0f2-4ba4-a999-fda22d821586",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {tsc_dir} && loompy fromfq TSP10_S6.loom TSP10_Skin_NA_10X_1_2_S6 {human_gencode_dir} metadata.tab \\\n",
    "Pilot_10/TSP10_Skin_NA_10X_1_2/TSP10_Skin_NA_10X_1_2_S6_L001_R1_001.fastq.gz Pilot_10/TSP10_Skin_NA_10X_1_2/TSP10_Skin_NA_10X_1_2_S6_L001_R2_001.fastq.gz \\\n",
    "Pilot_10/TSP10_Skin_NA_10X_1_2/TSP10_Skin_NA_10X_1_2_S6_L002_R1_001.fastq.gz Pilot_10/TSP10_Skin_NA_10X_1_2/TSP10_Skin_NA_10X_1_2_S6_L002_R2_001.fastq.gz \\\n",
    "Pilot_10/TSP10_Skin_NA_10X_1_2/TSP10_Skin_NA_10X_1_2_S6_L003_R1_001.fastq.gz Pilot_10/TSP10_Skin_NA_10X_1_2/TSP10_Skin_NA_10X_1_2_S6_L003_R2_001.fastq.gz \\\n",
    "Pilot_10/TSP10_Skin_NA_10X_1_2/TSP10_Skin_NA_10X_1_2_S6_L004_R1_001.fastq.gz Pilot_10/TSP10_Skin_NA_10X_1_2/TSP10_Skin_NA_10X_1_2_S6_L004_R2_001.fastq.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0921e362-735d-40a2-bc13-81a9eeb2e6d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {tsc_dir} && loompy fromfq TSP14_S17.loom TSP14_Skin_Abdomen_10X_1_1_S17 {human_gencode_dir} metadata.tab \\\n",
    "Pilot_14/TSP14_Skin_Abdomen_10X_1_1/TSP14_Skin_Abdomen_10X_1_1_S17_R1_001.fastq.gz Pilot_14/TSP14_Skin_Abdomen_10X_1_1/TSP14_Skin_Abdomen_10X_1_1_S17_R2_001.fastq.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "011992ba-6873-4379-a15d-946b93542ee8",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {tsc_dir} && loompy fromfq TSP14_S18.loom TSP14_Skin_Chest_10X_1_1_S18 {human_gencode_dir} metadata.tab \\\n",
    "Pilot_14/TSP14_Skin_Chest_10X_1_1/TSP14_Skin_Chest_10X_1_1_S18_R1_001.fastq.gz Pilot_14/TSP14_Skin_Chest_10X_1_1/TSP14_Skin_Chest_10X_1_1_S18_R2_001.fastq.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b75b54a7-df73-47e5-a0a4-9c805db221a0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_tsc_2021_ctrl_human_TSP10_S5 = sc.read_loom(f\"{tsc_dir}/TSP10_S5.loom\")\n",
    "adata_tsc_2021_ctrl_human_TSP10_S5.var_names_make_unique()\n",
    "metadata_assignment(adata_tsc_2021_ctrl_human_TSP10_S5, 'Tabula Sapiens', 2021, \"T10_S5\")\n",
    "\n",
    "adata_tsc_2021_ctrl_human_TSP10_S6 = sc.read_loom(f\"{tsc_dir}/TSP10_S6.loom\")\n",
    "adata_tsc_2021_ctrl_human_TSP10_S6.var_names_make_unique()\n",
    "metadata_assignment(adata_tsc_2021_ctrl_human_TSP10_S6, 'Tabula Sapiens', 2021, \"T10_S6\")\n",
    "\n",
    "adata_tsc_2021_ctrl_human_TSP14_S17 = sc.read_loom(f\"{tsc_dir}/TSP14_S17.loom\")\n",
    "adata_tsc_2021_ctrl_human_TSP14_S17.var_names_make_unique()\n",
    "metadata_assignment(adata_tsc_2021_ctrl_human_TSP14_S17, 'Tabula Sapiens', 2021, \"T14_S17\")\n",
    "\n",
    "adata_tsc_2021_ctrl_human_TSP14_S18 = sc.read_loom(f\"{tsc_dir}/TSP14_S18.loom\")\n",
    "adata_tsc_2021_ctrl_human_TSP14_S18.var_names_make_unique()\n",
    "metadata_assignment(adata_tsc_2021_ctrl_human_TSP14_S18, 'Tabula Sapiens', 2021, \"T14_S18\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "87ab735e-918c-4b8e-b7f5-6533e85f4181",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_tsc_2021_ctrl_human = sc.AnnData.concatenate(adata_tsc_2021_ctrl_human_TSP10_S5, adata_tsc_2021_ctrl_human_TSP10_S6, adata_tsc_2021_ctrl_human_TSP14_S17, adata_tsc_2021_ctrl_human_TSP14_S18, \n",
    "                                             batch_categories=['T10_S5', 'T10_S6', 'T14_S17', 'T14_S18'], batch_key=\"Internal sample identifier\")\n",
    "adata_tsc_2021_ctrl_human.write_h5ad(tsc_dir + '/adata_tsc_2021_ctrl_human.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "597e6cc2-aae3-4a9a-b929-f21a7b045ee8",
   "metadata": {},
   "source": [
    "## The Human Protein Atlas 2021 (human)\n",
    "\n",
    "The data are extracted from Solé-Boldo (GSE130973)."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "643811ec-cc65-4580-ad92-253cf894221b",
   "metadata": {},
   "source": [
    "## Theocharidis et al. 2020 (human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bcf74d50-3300-4a31-89a4-de1ebef5e706",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "theo_dir = data_dir + '/Theocharidis_2020/'\n",
    "os.makedirs(theo_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7fd69b39-8cf9-48ad-a33c-5b6686af7759",
   "metadata": {},
   "outputs": [],
   "source": [
    "# The file was obtained by personal request (https://www.dropbox.com/scl/fo/x4106l4nd2s8rrec4mboh/AACGrmqWuvbhXZaNnxOdZh9ja?dl=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "60b840e6-478a-4a7d-86ea-2c5eaa0daffe",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_theo_healthy_1 = sc.read_10x_mtx(theo_dir + 'Human samples raw_GT_Veves lab/Healthy/H1_080717')\n",
    "metadata_assignment(adata_theo_healthy_1, 'Theocarditis', 2020, \"H1\")\n",
    "adata_theo_healthy_2 = sc.read_10x_mtx(theo_dir + 'Human samples raw_GT_Veves lab/Healthy/H2_091117')\n",
    "metadata_assignment(adata_theo_healthy_2, 'Theocarditis', 2020, \"H2\")\n",
    "adata_theo_healthy_3 = sc.read_10x_mtx(theo_dir + 'Human samples raw_GT_Veves lab/Healthy/H3_091117')\n",
    "metadata_assignment(adata_theo_healthy_3, 'Theocarditis', 2020, \"H3\")\n",
    "adata_theo_healthy_4 = sc.read_10x_mtx(theo_dir + 'Human samples raw_GT_Veves lab/Healthy/H4_100317')\n",
    "metadata_assignment(adata_theo_healthy_4, 'Theocarditis', 2020, \"H4\")\n",
    "\n",
    "adata_theo_healthy = sc.AnnData.concatenate(adata_theo_healthy_1, adata_theo_healthy_2, adata_theo_healthy_3, adata_theo_healthy_4, batch_categories=['H1', 'H2', 'H3', 'H4'])\n",
    "\n",
    "sc.pp.filter_genes(adata_theo_healthy, min_counts=1)\n",
    "adata_theo_healthy.write_h5ad(theo_dir + '/adata_theo_ctrl_human.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "75e46bcd-8224-43b7-81b3-839c3a02e449",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_theo_dm_noDFU_1 = sc.read_10x_mtx(theo_dir + 'Human samples raw_GT_Veves lab/DM/DM1_091117')\n",
    "metadata_assignment(adata_theo_dm_noDFU_1, 'Theocarditis', 2020, \"DM1\")\n",
    "adata_theo_dm_noDFU_2 = sc.read_10x_mtx(theo_dir + 'Human samples raw_GT_Veves lab/DM/DM2_100317')\n",
    "metadata_assignment(adata_theo_dm_noDFU_2, 'Theocarditis', 2020, \"DM2\")\n",
    "adata_theo_dm_noDFU_3 = sc.read_10x_mtx(theo_dir + 'Human samples raw_GT_Veves lab/DM/DM3_100317')\n",
    "metadata_assignment(adata_theo_dm_noDFU_3, 'Theocarditis', 2020, \"DM3\")\n",
    "adata_theo_dm_noDFU_4 = sc.read_10x_mtx(theo_dir + 'Human samples raw_GT_Veves lab/DM/DM4_100317')\n",
    "metadata_assignment(adata_theo_dm_noDFU_4, 'Theocarditis', 2020, \"DM4\")\n",
    "\n",
    "adata_theo_dm_noDFU = sc.AnnData.concatenate(adata_theo_dm_noDFU_1, adata_theo_dm_noDFU_2, adata_theo_dm_noDFU_3, adata_theo_dm_noDFU_4, \n",
    "                                             batch_categories=['DM1', 'DM2', 'DM3', 'DM4'])\n",
    "\n",
    "sc.pp.filter_genes(adata_theo_dm_noDFU, min_counts=1)\n",
    "adata_theo_dm_noDFU.write_h5ad(theo_dir + '/adata_theo_dm_noDFU_human.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10792c49-79af-47f0-b574-4719960d8495",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_theo_dm_DFU_1 = sc.read_10x_mtx(theo_dir + 'Human samples raw_GT_Veves lab/DFU/DFU1_091117')\n",
    "metadata_assignment(adata_theo_dm_DFU_1, 'Theocarditis', 2020, \"DFU1\")\n",
    "adata_theo_dm_DFU_2 = sc.read_10x_mtx(theo_dir + 'Human samples raw_GT_Veves lab/DFU/DFU2_091117')\n",
    "metadata_assignment(adata_theo_dm_DFU_2, 'Theocarditis', 2020, \"DFU2\")\n",
    "adata_theo_dm_DFU_3 = sc.read_10x_mtx(theo_dir + 'Human samples raw_GT_Veves lab/DFU/DFU3_031418')\n",
    "metadata_assignment(adata_theo_dm_DFU_3, 'Theocarditis', 2020, \"DFU3\")\n",
    "adata_theo_dm_DFU_4 = sc.read_10x_mtx(theo_dir + 'Human samples raw_GT_Veves lab/DFU/DFU4_031418')\n",
    "metadata_assignment(adata_theo_dm_DFU_4, 'Theocarditis', 2020, \"DFU4\")\n",
    "\n",
    "adata_theo_dm_DFU = sc.AnnData.concatenate(adata_theo_dm_noDFU_1, adata_theo_dm_noDFU_2, adata_theo_dm_noDFU_3, adata_theo_dm_noDFU_4, \n",
    "                                             batch_categories=['DFU1', 'DFU2', 'DFU3', 'DFU4'])\n",
    "\n",
    "sc.pp.filter_genes(adata_theo_dm_DFU, min_counts=1)\n",
    "adata_theo_dm_DFU.write_h5ad(theo_dir + '/adata_theo_dm_DFU_human.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "570abd42-f1df-41c4-9412-0095bd302bc6",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Theocarditis et al. 2021 (human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f2140e9c-4f37-4f47-858e-7681d213a749",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "theo_dir_2021 = data_dir + 'Theocharidis_2021/'\n",
    "os.makedirs(theo_dir_2021, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dfcdc7b2-8d89-4ac7-933f-89f48a62d7c8",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "list_g = ['G1', 'G10', 'G14', 'G16', 'G18', 'G24', 'G28', 'G29', 'G31', 'G32', 'G36', 'G40', 'G43', 'G44', 'G50', 'G1A', 'G2A', 'G3', 'G3A', 'G4A', 'G5', 'G38', \n",
    " 'G41', 'G46', 'G48', 'G2', 'G4', 'G7', 'G8', 'G12', 'G13', 'G15', 'G17', 'G23', 'G42', 'G45', 'G49', 'G6', 'G9', 'G11', 'G33', 'G34', 'G35', 'G39']\n",
    "list_GSM = ['GSM5050521', 'GSM5050534', 'GSM5050538', 'GSM5050540', 'GSM5050542', 'GSM5050548', 'GSM5050552', 'GSM5050553', 'GSM5050555', 'GSM5050556', 'GSM5050560',\n",
    "            'GSM5050564', 'GSM5050567', 'GSM5050568', 'GSM5050574', 'GSM5050522', 'GSM5050524', 'GSM5050525', 'GSM5050526', 'GSM5050528', 'GSM5050529', 'GSM5050562',\n",
    "            'GSM5050565', 'GSM5050570', 'GSM5050572', 'GSM5050523', 'GSM5050527', 'GSM5050531', 'GSM5050532', 'GSM5050536', 'GSM5050537', 'GSM5050539', 'GSM5050541', \n",
    "            'GSM5050547', 'GSM5050566', 'GSM5050569', 'GSM5050573', 'GSM5050530', 'GSM5050533', 'GSM5050535', 'GSM5050557', 'GSM5050558', 'GSM5050559', 'GSM5050563', ]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "61d52c72-188d-4741-8bc2-e5e64dd1c984",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Download all files at once\n",
    "for g, GSM in zip(list_g, list_GSM):\n",
    "    os.system(f\"cd {theo_dir_2021} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM5050nnn/{GSM}/suppl/{GSM}%5F{g}counts%2Ecsv%2Egz -O {g}.csv.gz\")\n",
    "    os.system(f\"cd {theo_dir_2021} && gunzip {g}.csv.gz\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "31c21db8-0a1d-4421-800c-73751ec3e56a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "list_adatas = []\n",
    "\n",
    "for g in tqdm(list(list_g)):\n",
    "    adata = sc.read(theo_dir_2021 + f'{g}.csv').transpose()\n",
    "    adata.X = spr.csr.csr_matrix(adata.X).copy()\n",
    "    metadata_assignment(adata, 'Theocarditis', 2021, g)\n",
    "\n",
    "    \n",
    "    list_adatas.append(adata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a2f81545-a3b4-471a-bd24-569c33c5e41c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_theo_2021 = sc.AnnData.concatenate(*list_adatas, batch_categories=list_g)\n",
    "adata_theo_2021.write_h5ad(theo_dir_2021 + 'adata_theo_2021_human.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "effective-moderator",
   "metadata": {},
   "source": [
    "## Vorstandlechner et al. 2020 (human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "orange-nitrogen",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "vors_2020_dir = data_dir + '/Vorstandlechner_2020'\n",
    "os.makedirs(vors_2020_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "objective-stuff",
   "metadata": {},
   "outputs": [],
   "source": [
    "# The file was obtained by personal request from Vorstandlechner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4c7f22bf-2ad2-4c2f-9c31-db9d40897e83",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "vors_2020_ctrl_human = sc.read(vors_2020_dir + '/skin_vorstandlechner.loom', cache=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "235068a4-ca65-420c-a0e2-ff4996e260d1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "vors_2020_ctrl_human.obs['Internal sample identifier'] = [i.split('-')[1] for i in vors_2020_ctrl_human.obs_names]\n",
    "vors_2020_ctrl_human.obs['Internal sample identifier'] = vors_2020_ctrl_human.obs['Internal sample identifier'].astype('category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ed6b0875-a41a-4c8f-99d6-945d4e2b8dde",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "list_adatas = []\n",
    "\n",
    "for batch in ['1', '2', '3']:\n",
    "    adata = vors_2020_ctrl_human[vors_2020_ctrl_human.obs['Internal sample identifier'] == batch]\n",
    "    adata = metadata_assignment(adata, 'Vorstandlechner', 2020, batch, do_return=True)\n",
    "    list_adatas.append(adata)\n",
    "    \n",
    "vors_2020_ctrl_human = sc.AnnData.concatenate(*list_adatas, batch_key=\"Internal sample identifier\", batch_categories=['1', '2', '3'])\n",
    "vors_2020_ctrl_human.write_h5ad(vors_2020_dir + '/adata_vors_2020_ctrl_human.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cc1d68b3-8bb3-41d4-946d-8d9bb4da847f",
   "metadata": {},
   "source": [
    "## Vorstandlechner et al. 2021 (human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "829bae6b-445f-4300-bedf-6780793d1c7b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "vors_2021_dir = data_dir + '/Vorstandlechner_2021'\n",
    "os.makedirs(vors_2021_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fa3fbb61-459a-4455-a036-bbd50542a14a",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for gsm, name in zip(['GSM4729097', 'GSM4729098', 'GSM4729099', 'GSM4729100', 'GSM4729101', 'GSM4729102'], \n",
    "                     ['human_skin_1', 'human_skin_2', 'human_skin_3', 'human_scar_1', 'human_scar_2', 'human_scar_3']):\n",
    "    !cd {vors_2021_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4729nnn/{gsm}/suppl/{gsm}%5F{name.replace('_', '%5F')}%5Fbarcodes%2Etsv%2Egz\n",
    "    !cd {vors_2021_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4729nnn/{gsm}/suppl/{gsm}%5F{name.replace('_', '%5F')}%5Ffeatures%2Etsv%2Egz  # In Ctrl\n",
    "    !cd {vors_2021_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4729nnn/{gsm}/suppl/{gsm}%5F{name.replace('_', '%5F')}%5Fgenes%2Etsv%2Egz  # In scar\n",
    "    !cd {vors_2021_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4729nnn/{gsm}/suppl/{gsm}%5F{name.replace('_', '%5F')}%5Fmatrix%2Emtx%2Egz\n",
    "\n",
    "    !mkdir {vors_2021_dir}/{gsm}\n",
    "    !mv {vors_2021_dir}/{gsm}_{name}_barcodes.tsv.gz {vors_2021_dir}/{gsm}/barcodes.tsv.gz\n",
    "    !mv {vors_2021_dir}/{gsm}_{name}_features.tsv.gz {vors_2021_dir}/{gsm}/features.tsv.gz\n",
    "    !mv {vors_2021_dir}/{gsm}_{name}_genes.tsv.gz {vors_2021_dir}/{gsm}/features.tsv.gz\n",
    "    !mv {vors_2021_dir}/{gsm}_{name}_matrix.mtx.gz {vors_2021_dir}/{gsm}/matrix.mtx.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "135cb438-8213-421b-af34-ceb94da6d4c4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "list_adatas = []\n",
    "\n",
    "for gsm, name in zip(['GSM4729097', 'GSM4729098', 'GSM4729099',], \n",
    "                     ['human_skin_1', 'human_skin_2', 'human_skin_3',]):\n",
    "    adata = sc.read_mtx(vors_2021_dir + f'/{gsm}/matrix.mtx.gz').transpose()\n",
    "    adata.var_names = pd.read_csv(vors_2021_dir + f'/{gsm}/features.tsv.gz', sep='\\t', header=None)[1].values\n",
    "    adata.obs_names = pd.read_csv(vors_2021_dir + f'/{gsm}/barcodes.tsv.gz', header=None)[0].values\n",
    "    \n",
    "    adata.var_names_make_unique()\n",
    "    metadata_assignment(adata, 'Vorstandlechner', 2021, name)\n",
    "    list_adatas.append(adata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "85b1c83a-f0c8-4dd9-95d6-07f9279270b9",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "vorstandlechner_2021_ctrl_human = sc.AnnData.concatenate(*list_adatas, \n",
    "                                   batch_key=\"Internal sample identifier\", batch_categories=['human_skin_1', 'human_skin_2', 'human_skin_3', ])\n",
    "vorstandlechner_2021_ctrl_human.write_h5ad(vors_2021_dir + '/vorstandlechner_2021_ctrl_human.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "155e2a86-a8c9-440a-8a0d-d0df42a21e6a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "list_adatas = []\n",
    "\n",
    "for gsm, name in zip(['GSM4729100', 'GSM4729101', 'GSM4729102'], \n",
    "                     ['human_scar_1', 'human_scar_2', 'human_scar_3']):\n",
    "    adata = sc.read_mtx(vors_2021_dir + f'/{gsm}/matrix.mtx.gz').transpose()\n",
    "    adata.var_names = pd.read_csv(vors_2021_dir + f'/{gsm}/features.tsv.gz', sep='\\t', header=None)[1].values\n",
    "    adata.obs_names = pd.read_csv(vors_2021_dir + f'/{gsm}/barcodes.tsv.gz', header=None)[0].values\n",
    "    \n",
    "    adata.var_names_make_unique()\n",
    "    metadata_assignment(adata, 'Vorstandlechner', 2021, name)\n",
    "    list_adatas.append(adata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3a4f64e9-b53e-45da-8ec8-3eb24964515c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "vorstandlechner_2021_scar = sc.AnnData.concatenate(*list_adatas, \n",
    "                                   batch_key=\"Internal sample identifier\", batch_categories=['human_scar_1', 'human_scar_2', 'human_scar_3'])\n",
    "vorstandlechner_2021_scar.write_h5ad(vors_2021_dir + '/vorstandlechner_2021_scar_human.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f07fc6de-57e2-437f-9da4-73bf71b3d858",
   "metadata": {},
   "source": [
    "## Xu et al. 2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "25471a54-89ad-44ef-a37c-cf79071a823b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "xu_2021_dir = data_dir + '/xu_2021'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f100a89d-15dd-4601-ba8e-cdd4a86fd2dd",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {xu_2021_dir} && wget https://download.cncb.ac.cn/OMIX/OMIX691/OMIX691-20-01.tar.gz --no-check-certificate\n",
    "!cd {xu_2021_dir} && wget https://download.cncb.ac.cn/OMIX/OMIX691/OMIX691-20-02.tar.gz --no-check-certificate\n",
    "!cd {xu_2021_dir} && wget https://download.cncb.ac.cn/OMIX/OMIX691/OMIX691-20-03.tar.gz --no-check-certificate\n",
    "!cd {xu_2021_dir} && wget https://download.cncb.ac.cn/OMIX/OMIX691/OMIX691-20-04.tar.gz --no-check-certificate\n",
    "!cd {xu_2021_dir} && wget https://download.cncb.ac.cn/OMIX/OMIX691/OMIX691-20-05.tar.gz --no-check-certificate\n",
    "!cd {xu_2021_dir} && wget https://download.cncb.ac.cn/OMIX/OMIX691/OMIX691-20-06.tar.gz --no-check-certificate\n",
    "!cd {xu_2021_dir} && wget https://download.cncb.ac.cn/OMIX/OMIX691/OMIX691-20-07.tar.gz --no-check-certificate\n",
    "!cd {xu_2021_dir} && wget https://download.cncb.ac.cn/OMIX/OMIX691/OMIX691-20-08.tar.gz --no-check-certificate\n",
    "!cd {xu_2021_dir} && wget https://download.cncb.ac.cn/OMIX/OMIX691/OMIX691-20-09.tar.gz --no-check-certificate\n",
    "!cd {xu_2021_dir} && wget https://download.cncb.ac.cn/OMIX/OMIX691/OMIX691-20-10.tar.gz --no-check-certificate\n",
    "!cd {xu_2021_dir} && wget https://download.cncb.ac.cn/OMIX/OMIX691/OMIX691-20-11.tar.gz --no-check-certificate\n",
    "!cd {xu_2021_dir} && wget https://download.cncb.ac.cn/OMIX/OMIX691/OMIX691-20-12.tar.gz --no-check-certificate\n",
    "!cd {xu_2021_dir} && wget https://download.cncb.ac.cn/OMIX/OMIX691/OMIX691-20-13.tar.gz --no-check-certificate\n",
    "!cd {xu_2021_dir} && wget https://download.cncb.ac.cn/OMIX/OMIX691/OMIX691-20-14.tar.gz --no-check-certificate\n",
    "!cd {xu_2021_dir} && wget https://download.cncb.ac.cn/OMIX/OMIX691/OMIX691-20-15.tar.gz --no-check-certificate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eec2b7c5-c64a-4e16-a913-b90bc7301b1b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!cd {xu_2021_dir} && cat *.tar.gz | tar zxvf - -i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f8cde9dc-3f34-4fe5-aff9-ea69fcf143ab",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "list_adatas_healthy, list_adatas_disease = [], []\n",
    "list_names_healthy, list_names_disease = ['H01', 'H02', 'H03', 'H04', 'H05'], ['P01', 'P02', 'P03', 'P04', 'P05', 'P06', 'P07', 'P08', 'P09', 'P10']\n",
    "\n",
    "for name in list_names_healthy:\n",
    "    adata = sc.read_10x_mtx(xu_2021_dir + f'/{name}/outs/filtered_gene_bc_matrices/GRCh38')\n",
    "    adata.var_names_make_unique()\n",
    "    metadata_assignment(adata, 'Xu', 2021, name)\n",
    "    list_adatas_healthy.append(adata)\n",
    "    \n",
    "for name in list_names_disease:\n",
    "    adata = sc.read_10x_mtx(xu_2021_dir + f'/{name}/outs/filtered_gene_bc_matrices/GRCh38')\n",
    "    adata.var_names_make_unique()\n",
    "    metadata_assignment(adata, 'Xu', 2021, name)\n",
    "    list_adatas_disease.append(adata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ac59ab9c-0eb9-4afb-b7c9-0e48835f9277",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_xu_2021_healthy = sc.AnnData.concatenate(*list_adatas_healthy, batch_categories=list_names_healthy, batch_key='Internal sample identifier')\n",
    "adata_xu_2021_disease = sc.AnnData.concatenate(*list_adatas_disease, batch_categories=list_names_disease, batch_key='Internal sample identifier')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "70766020-e946-46d0-98cb-aea399329e81",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_xu_2021_healthy.write_h5ad(xu_2021_dir + '/xu_2021_healthy_human.h5')\n",
    "adata_xu_2021_disease.write_h5ad(xu_2021_dir + '/xu_2021_disease_human.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "15d82f37-81ad-4b9d-8f79-e0c170949efd",
   "metadata": {},
   "source": [
    "# Papers from other organs\n",
    "\n",
    "We are not going to process and apply metadata as in the skin samples, but we will apply a similar naming convention to keep track of the datasets and to unify the processing pipeline."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "025c2427-ee41-4bbb-9d4f-e8dd41418f9a",
   "metadata": {},
   "source": [
    "## Kuppe et al. 2021 [Kidney]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e875336b-70f8-44a2-bad6-7371543d050b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "kuppe_2021_dir = data_dir + '/kuppe_2021'\n",
    "os.makedirs(kuppe_2021_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "15bc6315-4f29-4cd0-82e5-b1f0584fa352",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!cd {kuppe_2021_dir} && wget https://zenodo.org/record/4059315/files/Human_PDGFRb.tar.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e2bfd2b8-69fc-4a7a-84c0-fee789e450bd",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!cd {kuppe_2021_dir} && tar xzvf {kuppe_2021_dir}/Human_PDGFRb.tar.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e616c072-7e17-4558-9c5c-d1e7e4f1253f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "kupper_2021_kidney_ctrl = sc.read(f\"{kuppe_2021_dir}/Human_PDGFRb/pdgfrbMap_UMI_counts.mtx\").transpose()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d0bf6821-8822-4873-9c40-79caf1f539c7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "kupper_2021_kidney_ctrl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0289df3a-54b1-49f6-9fea-8ae6703eec09",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "kupper_2021_kidney_ctrl_genes = pd.read_csv(f\"{kuppe_2021_dir}/Human_PDGFRb/pdgfrbMap_UMI_counts_rowData.txt\")\n",
    "kupper_2021_kidney_ctrl_meta = pd.read_csv(f\"{kuppe_2021_dir}/Human_PDGFRb/pdgfrbMap_UMI_counts_colData.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "36b43ae3-2abf-4f9d-87f8-27a43d8aab78",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "kupper_2021_kidney_ctrl.var_names = kupper_2021_kidney_ctrl_genes['Gene.Symbol'].values\n",
    "kupper_2021_kidney_ctrl.var_names_make_unique()\n",
    "\n",
    "kupper_2021_kidney_ctrl.obs =  kupper_2021_kidney_ctrl_meta\n",
    "kupper_2021_kidney_ctrl.obs['Internal sample identifier'] = [i.split('_')[0] for i in kupper_2021_kidney_ctrl.obs['Patient ID']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7902af61-75ac-4dd8-926f-fd5377469c95",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "kupper_2021_kidney_ctrl.X = spr.csr_matrix(kupper_2021_kidney_ctrl.X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9273887d-e150-4323-87b1-7112e2d8bf16",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "kupper_2021_kidney_ctrl.write_h5ad(kuppe_2021_dir + '/kupper_2021_kidney_pdgfrb_ctrl.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7010f124-6faa-4e5d-8393-02996553a32d",
   "metadata": {},
   "source": [
    "## Travaglini et al. 2020 [Lung]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fa3ee2e3-a453-45ab-8742-6efdeca93420",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "travaglini_2020_dir = data_dir + '/travaglini_2020'\n",
    "os.makedirs(travaglini_2020_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b8654632-1b9b-4644-bd42-ffcb7674f1b1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# To download the data you have to register in Synapse and download it from there [https://www.synapse.org/#!Synapse:syn21041850/wiki/600865]\n",
    "travaglini_2020_lung_ctrl = sc.read(f\"{travaglini_2020_dir}/krasnow_hlca_10x_UMIs.csv\").transpose()\n",
    "travaglini_2020_lung_ctrl.var_names_make_unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4b1c4e37-f3ee-4182-afcd-62fb946810cf",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "travaglini_2020_lung_ctrl.X = spr.csr_matrix(travaglini_2020_lung_ctrl.X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dd6db1f2-74ba-4342-9e04-54d493c4ecfe",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "travaglini_2020_lung_ctrl.obs['Internal sample identifier'] = [i.split('_')[0] for i in travaglini_2020_lung_ctrl.obs_names]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "29cd39ba-2624-4471-80e2-75e7ada9fee4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "travaglini_2020_lung_ctrl.obs['Internal sample identifier']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1756fef5-29af-4fbf-93f9-582c07930d01",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "travaglini_2020_lung_ctrl.write_h5ad(travaglini_2020_dir + '/travaglini_2020_lung_ctrl.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "40356b28-f8a9-4f28-8401-6c6490b9d596",
   "metadata": {},
   "source": [
    "## Litviňuková et al. 2020 [Heart]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b9fb9db3-b3f6-4689-9696-29d96c31a45d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "litvinukova_2020_dir = data_dir + '/litvinukova_2020'\n",
    "os.makedirs(litvinukova_2020_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bc2d5f13-2a41-43b5-aa9d-8c60a6c29621",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!cd {litvinukova_2020_dir} && wget https://cellgeni.cog.sanger.ac.uk/heartcellatlas/data/global_raw.h5ad\n",
    "litvinukova_2020_heart_ctrl = sc.read(f\"{litvinukova_2020_dir}/global_raw.h5ad\")\n",
    "litvinukova_2020_heart_ctrl.var_names_make_unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb84df5e-9cfe-4cfc-97cd-30fda9c30ac4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "litvinukova_2020_heart_ctrl = litvinukova_2020_heart_ctrl[litvinukova_2020_heart_ctrl.obs['source'] == 'Cells']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "64968301-a94c-4ee1-a840-5add07b460e7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "litvinukova_2020_heart_ctrl.obs['Internal sample identifier'] = litvinukova_2020_heart_ctrl.obs['donor']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "de71185f-d64a-4b75-853f-3f99c0e65eb2",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "litvinukova_2020_heart_ctrl.write_h5ad(litvinukova_2020_dir + '/litvinukova_2020_heart_ctrl.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "96f6b324-30da-40cc-9447-254a242007bd",
   "metadata": {},
   "source": [
    "## Elmentaite et al. 2021 [Instesinal tract]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5e1a98e3-2f05-4bed-9cb2-dd95cebef148",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "elmentaite_2021_dir = data_dir + '/elmentaite_2021'\n",
    "os.makedirs(elmentaite_2021_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "de40b7ee-12c4-44c4-9339-a78e0ea4bbe2",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!cd {elmentaite_2021_dir} && wget https://cellgeni.cog.sanger.ac.uk/gutcellatlas/Full_obj_raw_counts_nosoupx.h5ad\n",
    "elmentaite_2021_intestine_ctrl = sc.read(f\"{elmentaite_2021_dir}/Full_obj_raw_counts_nosoupx.h5ad\")\n",
    "elmentaite_2021_intestine_ctrl.var_names_make_unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b2af272-9da0-499b-b6b5-d0e3e396a56f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "elmentaite_2021_intestine_ctrl = elmentaite_2021_intestine_ctrl[elmentaite_2021_intestine_ctrl.obs['Diagnosis'] == 'Healthy adult']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "28831377-3314-4bfd-9221-6c2fae864e7e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "names, counts = np.unique(elmentaite_2021_intestine_ctrl.obs['batch'], return_counts =True)\n",
    "elmentaite_2021_intestine_ctrl = elmentaite_2021_intestine_ctrl[elmentaite_2021_intestine_ctrl.obs['batch'].isin(names[counts >= 100])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cf8d8558-540c-45e8-9fae-4954062a87cb",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "elmentaite_2021_intestine_ctrl.obs['Internal sample identifier'] = elmentaite_2021_intestine_ctrl.obs['batch']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0be4537d-4b07-43ad-ace9-9141a635f803",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "elmentaite_2021_intestine_ctrl.write_h5ad(elmentaite_2021_dir + '/elmentaite_2021_intestine_ctrl.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3131be61-3343-42bc-9e4c-fbe4fd61f7d8",
   "metadata": {},
   "source": [
    "## Ramachandran et al. 2019 [Liver]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9eb63e72-dfd1-468a-b1b8-57794ca3945b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "ramachandran_2019_dir = data_dir + '/ramachandran_2019'\n",
    "os.makedirs(ramachandran_2019_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f4fe5161-b001-4747-95e9-a1965e35f077",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "gsm_dict = {'GSM4041151': 'healthy1_cd45-A',  'GSM4041152': 'healthy1_cd45-B', 'GSM4041154': 'healthy2_cd45-', 'GSM4041156': 'healthy3_cd45-A', 'GSM4041157': 'healthy3_cd45-B', 'GSM4041159': 'healthy4_cd45-'}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "67c3f965-4822-44ad-b74e-4f58cb55e0d3",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for gsm, name in gsm_dict.items():    \n",
    "                \n",
    "    !cd {ramachandran_2019_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4041nnn/{gsm}/suppl/{gsm}%5F{name.replace('_', '%5F').replace('-', '%2D')}%5Fbarcodes%2Etsv%2Egz\n",
    "    !cd {ramachandran_2019_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4041nnn/{gsm}/suppl/{gsm}%5F{name.replace('_', '%5F').replace('-', '%2D')}%5Fgenes%2Etsv%2Egz  \n",
    "    !cd {ramachandran_2019_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4041nnn/{gsm}/suppl/{gsm}%5F{name.replace('_', '%5F').replace('-', '%2D')}%5Fmatrix%2Emtx%2Egz\n",
    "\n",
    "    !mkdir {ramachandran_2019_dir}/{gsm}\n",
    "    !mv {ramachandran_2019_dir}/{gsm}_{name}_barcodes.tsv.gz {ramachandran_2019_dir}/{gsm}/barcodes.tsv.gz\n",
    "    !mv {ramachandran_2019_dir}/{gsm}_{name}_genes.tsv.gz {ramachandran_2019_dir}/{gsm}/features.tsv.gz\n",
    "    !mv {ramachandran_2019_dir}/{gsm}_{name}_matrix.mtx.gz {ramachandran_2019_dir}/{gsm}/matrix.mtx.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c788c338-29d2-4c82-866c-b1f9211ef608",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "list_adatas = []\n",
    "\n",
    "for gsm, name in gsm_dict.items():\n",
    "    adata = sc.read_mtx(ramachandran_2019_dir + f'/{gsm}/matrix.mtx.gz').transpose()\n",
    "    adata.var_names = pd.read_csv(ramachandran_2019_dir + f'/{gsm}/features.tsv.gz', sep='\\t', header=None)[1].values\n",
    "    adata.obs_names = pd.read_csv(ramachandran_2019_dir + f'/{gsm}/barcodes.tsv.gz', header=None)[0].values\n",
    "    \n",
    "    adata.var_names_make_unique()\n",
    "    list_adatas.append(adata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2d439ce7-5eaf-4f21-a79f-2fd9ea4a8a3e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "ramachandran_2019_liver_ctrl = sc.AnnData.concatenate(*list_adatas, \n",
    "                                   batch_key=\"Internal sample identifier\", batch_categories=gsm_dict.values())\n",
    "ramachandran_2019_liver_ctrl.write_h5ad(ramachandran_2019_dir + '/ramachandran_2019_liver_ctrl.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f7618a3a-c636-4b75-b83a-2bb13083a14c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "290e02f2-063a-4a2c-8f7a-b8f7525fbeca",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dcb159e1-e526-4afe-852c-4ab82941143d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "15da52f9-09de-4d52-ada1-b22520f66281",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fbad7b82-56d5-480e-963c-12f179de0a9d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a2571082-02b9-4ea8-b97c-8aea321c11a2",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:alex-base] *",
   "language": "python",
   "name": "conda-env-alex-base-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}