[GC] Evaluating Explanations.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "684b54d2",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_12326/2313208493.py:6: DeprecationWarning: Importing display from IPython.core.display is deprecated since IPython 7.14, please import from IPython display\n",
      "  from IPython.core.display import display, HTML\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>#notebook { padding-top:0px !important; } .container { width:100% !important;ù } .end_space { min-heig§ht:0px !important; } </style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import networkx as nx\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "\n",
    "from IPython.core.display import display, HTML\n",
    "display(HTML(\n",
    "    '<style>'\n",
    "        '#notebook { padding-top:0px !important; } ' \n",
    "        '.container { width:100% !important;ù } '\n",
    "        '.end_space { min-heig§ht:0px !important; } '\n",
    "    '</style>'\n",
    "))\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6ea9808c",
   "metadata": {},
   "source": [
    "to simplify the analisys we are producing a section for each dataset! then for each of them we compute\n",
    "1. Filter\n",
    "2. Plausibility\n",
    "3. F1-fidelity\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7575010a",
   "metadata": {},
   "source": [
    "# GRID"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "cc666449",
   "metadata": {},
   "outputs": [],
   "source": [
    "from metrics.load_expl import load_graphs"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ca49afe1",
   "metadata": {},
   "source": [
    "the function load_graphs takes in input:\n",
    "1. Dataset name\n",
    "2. Architecture name\n",
    "3. Mode can be train or test\n",
    "4. verbose = True or False\n",
    "5. lamb it is a float\n",
    "6. normalize = True or False\n",
    "\n",
    "The output is a dictionary like:\n",
    "\n",
    "```\n",
    "graphs = {0: \n",
    "              {gid: nx.graph, gid:nx.graphs ...},\n",
    "          1: \n",
    "              {gid: nx.graph, gid:nx.graphs ...}\n",
    "          }\n",
    "            \n",
    "```\n",
    "The first key of the dictionary is the class (if binary it could be 0 or 1). The inner dictionary has as key the id of the graph,\n",
    "while the value is the explanation loaded with a networkx graph\n",
    "\n",
    "The lamb parameter is used to filter the explanation (as described in the paper).   \n",
    "If verbose = True, we can see the percentuage of graph that pass the filter for each class\n",
    "if normalize = True, then we nomalize the weigths from 0 to 1, and if the original explanatio is an edge mask, then we convert it into a node importance,\n",
    "simply averaging the neighbours edge weights. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "53bb30dc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.896 , 0.884\n"
     ]
    }
   ],
   "source": [
    "dataset = \"BA_2grid\"\n",
    "model_architecture = \"Cheb\"\n",
    "expl = \"grad_cam\"\n",
    "MODE = \"train\"\n",
    "\n",
    "graphs = load_graphs(DATASET=dataset,\n",
    "                 MODEL=model_architecture,\n",
    "                 EXPL=expl,\n",
    "                 MODE=MODE,\n",
    "                 verbose=True,\n",
    "                 lamb=0.001,\n",
    "                 normalize=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "05bcee6f",
   "metadata": {},
   "source": [
    "in this example, after we apply the filter the percentuage of graph that pass the filter in calss 0 is equal to 0.896."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "86a8f065",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{0: 0.005213894009245032, 1: 0.004552462656866181, 2: 0.002753594663883456, 3: 0.006146964621425074, 4: 0.002318356249601254, 5: 0.0029476325121268105, 6: 0.004414776083295482, 7: 0.0027290683235523285, 8: 0.0021135551940904755, 9: 0.0031675635979068915, 10: 0.004424907257441474, 11: 0.0031675635979068915, 12: 0.004452885671505211, 13: 0.004364849906167776, 14: 0.004452885671505211, 15: 0.0030158253199002666, 16: 0.005368853038781566, 17: 0.0030158253199002666}\n",
      "{0: 0.0047663727899846225, 1: 0.004044941011612636, 2: 0.0020828922962523003, 3: 0.005784085041861859, 4: 0.0016081720972443096, 5: 0.0022945319009160616, 6: 0.003894764475118248, 7: 0.0020561410978725972, 8: 0.0013847929224316627, 9: 0.0025344135965190335, 10: 0.0039058146785215984, 11: 0.0025344135965190335, 12: 0.003936331098617857, 13: 0.003840309344403601, 14: 0.003936331098617857, 15: 0.002368910683245093, 16: 0.004935388618840143, 17: 0.002368910683245093}\n"
     ]
    }
   ],
   "source": [
    "g = graphs[1][42]\n",
    "\n",
    "print(nx.get_node_attributes(g,\"node_imp\")) # original node importances\n",
    "print(nx.get_node_attributes(g,\"node_imp_norm\")) # normalized node importances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "83c38046",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "node impo     :  {0: 0.8080612199274574, 1: 0.6452593475875359, 2: 0.5095923425487956, 3: 0.7504838622401958, 4: 0.6932614486260131, 5: 0.7812933578528267, 6: 0.6202995412259948, 7: 0.5266569198385517, 8: 0.6240552652038245, 9: 0.7494803620021674, 10: 0.8585168098862009, 11: 0.7887489066297559, 12: 0.8598150678027413, 13: 0.9460512048975063, 14: 0.8105220974463769, 15: 0.8322114391720363, 16: 0.8397070296988869, 17: 0.7858296279556722}\n",
      "node impo norm:  {0: 0.7489275704172234, 1: 0.5359687216970191, 2: 0.3585046127950994, 3: 0.6736114353556495, 4: 0.598759597664103, 5: 0.7139129049948728, 6: 0.5033191486261719, 7: 0.3808265473115261, 8: 0.5082319585523348, 9: 0.6722987707510633, 10: 0.8149278208730523, 11: 0.7236654039145234, 12: 0.8166260538680087, 13: 0.9294303368275697, 14: 0.7521466098283691, 15: 0.780518133920865, 16: 0.7903230108926929, 17: 0.7198467363734754}\n",
      "edge impo     :  {(0, 1): 0.7809263526262762, (0, 3): 0.9499958585300826, (0, 4): 0.6932614486260131, (1, 2): 0.5095923425487956, (3, 5): 0.713942162613438, (3, 6): 0.713942162613438, (3, 8): 0.6240552652038245, (5, 16): 0.8486445530922154, (6, 7): 0.5266569198385517, (9, 10): 0.7494803620021674, (9, 12): 0.7494803620021674, (10, 11): 0.8280174512573444, (10, 13): 0.9980526163990908, (11, 14): 0.7494803620021674, (12, 13): 1.0, (12, 15): 0.8299648414060563, (13, 14): 0.8521210889309069, (13, 16): 0.9340311142600274, (14, 17): 0.8299648414060563, (15, 16): 0.8344580369380165, (16, 17): 0.7416944145052882}\n"
     ]
    }
   ],
   "source": [
    "# if we load explanations from an explaier with edge mask ... \n",
    "\n",
    "dataset = \"BA_2grid\"\n",
    "model_architecture = \"HO\"\n",
    "expl = \"ig_edge\"\n",
    "MODE = \"train\"\n",
    "\n",
    "graphs = load_graphs(DATASET=dataset,\n",
    "                 MODEL=model_architecture,\n",
    "                 EXPL=expl,\n",
    "                 MODE=MODE,\n",
    "                 verbose=False,\n",
    "                 lamb=0.001,\n",
    "                 normalize=True)\n",
    "\n",
    "g = graphs[1][42]\n",
    "\n",
    "print(\"node impo     : \",nx.get_node_attributes(g,\"node_imp\")) \n",
    "print(\"node impo norm: \",nx.get_node_attributes(g,\"node_imp_norm\")) # normalized node importances\n",
    "print(\"edge impo     : \",nx.get_edge_attributes(g,\"edge_imp\")) # original edge importances"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2f0604b1",
   "metadata": {},
   "source": [
    "### plausibility on GRID"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "7595fd25",
   "metadata": {},
   "outputs": [],
   "source": [
    "from metrics.plausibility import get_plausibility"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "d3990637",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9751295553085737\n"
     ]
    }
   ],
   "source": [
    "if not graphs[1] == None:\n",
    "    plau = get_plausibility(graphs,GT_len=9) \n",
    "print(plau)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f0e92254",
   "metadata": {},
   "source": [
    "### fidelity on GRID"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "7568c8d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "from metrics.F1_fidelity import build_expl,compute_fidelity\n",
    "from Datasets.synthetics import BA_2grid as dataset_fun\n",
    "\n",
    "\n",
    "DATASET = \"BA_2grid\"\n",
    "MODE = \"train\"\n",
    "MODEL = \"GCN\" \n",
    "expl = \"sal_edge\"\n",
    "\n",
    "# pay attention to load the correct model\n",
    "from models.models_BA_2grid import GCN_framework as framework"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "bda91c65",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "models/BA_2grid_GCN\n",
      "Test Loss: 0.003, Train Acc: 0.994 Test Acc: 0.998\n"
     ]
    }
   ],
   "source": [
    "gcn,graphs = build_expl(DATASET,MODEL,dataset_fun,framework,expl=expl,MODE=MODE)\n",
    "\n",
    "suf1, comp1 = compute_fidelity(gcn,graphs,y=1,color=False)\n",
    "suf0, comp0 = compute_fidelity(gcn,graphs,y=0,color=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "4fae74e2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "f1 class 0 : 0.34425428130625607\n",
      "f1 class 1 : 0.7225875146072238\n"
     ]
    }
   ],
   "source": [
    "f1_1 = 2*(((1-suf1) * (comp1))/((1-suf1) + (comp1)))\n",
    "f1_0 = 2*(((1-suf0) * (comp0))/((1-suf0) + (comp0)))\n",
    "\n",
    "print(\"f1 class 0 :\", f1_0)\n",
    "print(\"f1 class 1 :\", f1_1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dea2f4af",
   "metadata": {},
   "source": [
    "# GRID HOUSE \n",
    "### plausibility"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "e5a56d72",
   "metadata": {},
   "outputs": [],
   "source": [
    "from metrics.plausibility import get_plausibility_ba_grid_house,split_class0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "a57657ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = \"BA_2grid_house\"\n",
    "MODEL = \"GCN\" \n",
    "MODE = \"train\"\n",
    "expl = \"sal_edge\"\n",
    "\n",
    "graphs = load_graphs(DATASET=dataset,\n",
    "                     MODEL=MODEL,\n",
    "                     EXPL=expl,\n",
    "                     MODE=MODE,\n",
    "                     verbose=False,\n",
    "                     lamb=0.001,\n",
    "                     normalize=True)\n",
    "\n",
    "if not graphs[0] == None:\n",
    "    split_house_grid = split_class0(graphs[0])\n",
    "    acc = get_plausibility_ba_grid_house(split_house_grid[\"house\"],GT_len = 5)\n",
    "    acc1 = get_plausibility_ba_grid_house(split_house_grid[\"grid\"],GT_len = 9)\n",
    "    roc_class0 = np.mean([acc,acc1])\n",
    "\n",
    "if not graphs[1] == None:\n",
    "    roc_class1 = get_plausibility(graphs,GT_len = 9+5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "b45b8052",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pla class 0 0.849453761369126\n",
      "pla class 1 0.8381554258364556\n"
     ]
    }
   ],
   "source": [
    "print(\"pla class 0\",roc_class0)\n",
    "print(\"pla class 1\",roc_class1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6ea2ad38",
   "metadata": {},
   "source": [
    "### fidelity GRID HOUSE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "1361ade4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "models/BA_2grid_house_GCN\n",
      "Test Loss: 0.029, Train Acc: 0.971 Test Acc: 0.965\n"
     ]
    }
   ],
   "source": [
    "from metrics.F1_fidelity import build_expl,compute_fidelity\n",
    "from Datasets.synthetics import BA_2grid_house as dataset_fun\n",
    "\n",
    "DATASET = \"BA_2grid_house\"\n",
    "MODE = \"train\"\n",
    "expl = \"ig_edge\"\n",
    "MODEL = \"GCN\"\n",
    "from models.models_BA_2grid_house import GCN_framework as framework\n",
    "\n",
    "gcn,graphs = build_expl(DATASET,MODEL,dataset_fun,framework,expl=expl,MODE=MODE)\n",
    "\n",
    "if not graphs[1] == None:\n",
    "    suf1, comp1 = compute_fidelity(gcn,graphs,y=1)\n",
    "    f1_1 = 2*(((1-suf1) * (comp1))/((1-suf1) + (comp1)))\n",
    "if not graphs[0] == None:\n",
    "    suf0, comp0 = compute_fidelity(gcn,graphs,y=0)\n",
    "    f1_0 = 2*(((1-suf0) * (comp0))/((1-suf0) + (comp0)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "714ccab1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "f1 class 0 : 0.5770450399199949\n",
      "f1 class 1 : 0.4319210863728599\n"
     ]
    }
   ],
   "source": [
    "print(\"f1 class 0 :\", f1_0)\n",
    "print(\"f1 class 1 :\", f1_1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eff5250c",
   "metadata": {},
   "source": [
    "# STARS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "5db1dc46",
   "metadata": {},
   "outputs": [],
   "source": [
    "from metrics.plausibility import get_plausibility_nb_stars"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4c244632",
   "metadata": {},
   "source": [
    "### plausibility"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "b61de7d6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.000 , 1.000 , 1.000\n",
      "pla class 0 0.7189530392060536\n",
      "pla class 1 0.8912395781100527\n",
      "pla class 2 0.6196310958543689\n"
     ]
    }
   ],
   "source": [
    "dataset = \"ER_nb_stars2\"\n",
    "model_architecture = \"GCN\" \n",
    "MODE = \"train\"\n",
    "expl = \"sal_edge\"\n",
    "\n",
    "graphs = load_graphs(DATASET=dataset,MODEL=model_architecture,EXPL = expl,MODE=MODE)\n",
    "\n",
    "roc0 = get_plausibility_nb_stars(list(graphs[0].values()),1)\n",
    "roc1 = get_plausibility_nb_stars(list(graphs[1].values()),2)\n",
    "roc2 = get_plausibility_nb_stars(list(graphs[2].values()),3)\n",
    "\n",
    "print(\"pla class 0\",roc0)\n",
    "print(\"pla class 1\",roc1)\n",
    "print(\"pla class 2\",roc2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b7aaab62",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "### fidelity STARS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "08a153e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "from Datasets.synthetics import ER_nb_stars2 as dataset_fun\n",
    "\n",
    "dataset = \"ER_nb_stars2\"\n",
    "model_architecture = \"GCN\" \n",
    "MODE = \"train\"\n",
    "expl = \"sal_edge\"\n",
    "\n",
    "from models.models_ER_nb_stars2 import GCN_framework as framework"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "d52abc16",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "models/ER_nb_stars2_GCN\n",
      "Test Loss: 0.002, Train Acc: 0.994 Test Acc: 1.000\n"
     ]
    }
   ],
   "source": [
    "gcn,graphs = build_expl(dataset,model_architecture,dataset_fun,framework,expl=expl,MODE=MODE)\n",
    "suf1, comp1 = compute_fidelity(gcn,graphs,y=1)\n",
    "f1_1_tmp = 2*(((1-suf1) * (comp1))/((1-suf1) + (comp1)))\n",
    "\n",
    "suf0, comp0 = compute_fidelity(gcn,graphs,y=0)\n",
    "f1_0_tmp = 2*(((1-suf0) * (comp0))/((1-suf0) + (comp0)))\n",
    "\n",
    "suf2, comp2 = compute_fidelity(gcn,graphs,y=2)\n",
    "f1_2_tmp = 2*(((1-suf2) * (comp2))/((1-suf2) + (comp2)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "43b03881",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "f1 class 0 : 0.0115249958049668\n",
      "f1 class 1 : 0.27435323177693643\n",
      "f1 class 2 : 0.7929425627225647\n"
     ]
    }
   ],
   "source": [
    "print(\"f1 class 0 :\", f1_0_tmp)\n",
    "print(\"f1 class 1 :\", f1_1_tmp)\n",
    "print(\"f1 class 2 :\", f1_2_tmp)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6fadccb2",
   "metadata": {},
   "source": [
    "# HOUSE COLOR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "913b35fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "from metrics.load_expl import load_graphs\n",
    "from metrics.plausibility import load_gt_BA_houses_color,get_plausibility_houses_color"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "965ba119",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.000 , 1.000\n",
      "pla class 0 0.021089005798699493\n",
      "pla class 1 0.06051810194793691\n"
     ]
    }
   ],
   "source": [
    "dataset = \"BA_houses_color\"\n",
    "MODELS = \"GAT\"\n",
    "expl = \"cam\"\n",
    "MODE = \"train\"\n",
    "\n",
    "graphs = load_graphs(DATASET=dataset,MODEL=model_architecture,EXPL = expl,MODE=MODE)\n",
    "if MODE == \"train\":\n",
    "    graphs = load_gt_BA_houses_color(graphs,train=True)\n",
    "else:\n",
    "    graphs = load_gt_BA_houses_color(graphs,train=False)\n",
    "    \n",
    "roc0 = get_plausibility_houses_color(graphs,0)\n",
    "roc1 = get_plausibility_houses_color(graphs,1)\n",
    "\n",
    "print(\"pla class 0\",roc0)\n",
    "print(\"pla class 1\",roc1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "84a6458b",
   "metadata": {},
   "source": [
    "### fidelity house color"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "85c5b6b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "from metrics.F1_fidelity import build_expl,compute_fidelity\n",
    "from Datasets.synthetics import BA_houses_color as dataset_fun"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "bdd32510",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "models/BA_houses_color_GCN\n",
      "Test Loss: 0.005, Train Acc: 0.988 Test Acc: 0.995\n"
     ]
    }
   ],
   "source": [
    "DATASET = \"BA_houses_color\"\n",
    "MODEL = \"GCN\"\n",
    "MODE = \"train\"\n",
    "expl = \"ig_edge\"\n",
    "\n",
    "from models.models_BA_houses_color import GCN_framework as framework\n",
    "\n",
    "gcn,graphs = build_expl(DATASET,MODEL,dataset_fun,framework,expl=expl,MODE=MODE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "3b5331de",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "suf1, comp1 = compute_fidelity(gcn,graphs,y=1,color=True)\n",
    "f1_1_tmp = 2*(((1-suf1) * (comp1))/((1-suf1) + (comp1)))\n",
    "suf0, comp0 = compute_fidelity(gcn,graphs,y=0,color=True)\n",
    "f1_0_tmp = 2*(((1-suf0) * (comp0))/((1-suf0) + (comp0)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "06d1938f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "f1 class 0 : 0.15038511661447607\n",
      "f1 class 1 : 0.2404041806111522\n"
     ]
    }
   ],
   "source": [
    "print(\"f1 class 0 :\", f1_0_tmp)\n",
    "print(\"f1 class 1 :\", f1_1_tmp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "639baa0f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}