notebooks/gpt2/text_classification_linear_probe.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "1938fc31",
   "metadata": {
    "papermill": {
     "duration": 0.003654,
     "end_time": "2024-03-23T23:23:33.349348",
     "exception": false,
     "start_time": "2024-03-23T23:23:33.345694",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Linear probing of sentiment classification in a transformer trained on causal_lm\n",
    "In this notebook we'll try to find out if and how a transformer trained to do causal_lm process information about the sentiment of a sentence. We'll use the imdb dataset for this."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fbccdece",
   "metadata": {},
   "source": [
    "**GPU Requirements:** For running with GPT-2 you may be fine with just 8GB of GPU RAM. With about 24GB you should be able to run any 7B or 13B model. With 80GB (A100) GPU you may be able to run a 70B model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "39794270-9b0e-4aa8-8e5e-9a791972482d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-23T23:23:33.356520Z",
     "iopub.status.busy": "2024-03-23T23:23:33.356211Z",
     "iopub.status.idle": "2024-03-23T23:23:38.690114Z",
     "shell.execute_reply": "2024-03-23T23:23:38.689558Z"
    },
    "papermill": {
     "duration": 5.339026,
     "end_time": "2024-03-23T23:23:38.691626",
     "exception": false,
     "start_time": "2024-03-23T23:23:33.352600",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from transformer_heads import load_headed\n",
    "from datasets import load_dataset\n",
    "from transformers import (\n",
    "    AutoTokenizer,\n",
    "    MistralForCausalLM,\n",
    "    Trainer,\n",
    "    BitsAndBytesConfig,\n",
    "    TrainingArguments,\n",
    "    GPT2Model,\n",
    "    GPT2LMHeadModel,\n",
    ")\n",
    "from transformer_heads.util.helpers import DataCollatorWithPadding, get_model_params\n",
    "from peft import LoraConfig\n",
    "from transformer_heads.config import HeadConfig\n",
    "from transformer_heads.util.model import print_trainable_parameters\n",
    "from transformer_heads.util.evaluate import (\n",
    "    evaluate_head_wise,\n",
    "    get_top_n_preds,\n",
    "    get_some_preds,\n",
    ")\n",
    "import torch\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "626c412b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-23T23:23:38.699187Z",
     "iopub.status.busy": "2024-03-23T23:23:38.698828Z",
     "iopub.status.idle": "2024-03-23T23:23:38.702867Z",
     "shell.execute_reply": "2024-03-23T23:23:38.702376Z"
    },
    "papermill": {
     "duration": 0.008981,
     "end_time": "2024-03-23T23:23:38.704061",
     "exception": false,
     "start_time": "2024-03-23T23:23:38.695080",
     "status": "completed"
    },
    "tags": [
     "parameters"
    ]
   },
   "outputs": [],
   "source": [
    "# GPT2 is the fastest and requires fewest memory. However, this works just the same with any Llama or Mistral model. Just change model_path to its huggingface path.\n",
    "model_path = \"gpt2\"\n",
    "train_epochs = 1\n",
    "eval_epochs = 1\n",
    "logging_steps = 100\n",
    "full_finetune = False\n",
    "num_heads = 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1a1f7179",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "711228b7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-23T23:23:38.723024Z",
     "iopub.status.busy": "2024-03-23T23:23:38.722622Z",
     "iopub.status.idle": "2024-03-23T23:23:39.217041Z",
     "shell.execute_reply": "2024-03-23T23:23:39.216573Z"
    },
    "papermill": {
     "duration": 0.499101,
     "end_time": "2024-03-23T23:23:39.218167",
     "exception": false,
     "start_time": "2024-03-23T23:23:38.719066",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'vocab_size': 50257, 'n_positions': 1024, 'n_embd': 768, 'n_layer': 12, 'n_head': 12, 'n_inner': None, 'activation_function': 'gelu_new', 'resid_pdrop': 0.1, 'embd_pdrop': 0.1, 'attn_pdrop': 0.1, 'layer_norm_epsilon': 1e-05, 'initializer_range': 0.02, 'summary_type': 'cls_index', 'summary_use_proj': True, 'summary_activation': None, 'summary_first_dropout': 0.1, 'summary_proj_to_labels': True, 'scale_attn_weights': True, 'use_cache': True, 'scale_attn_by_inverse_layer_idx': False, 'reorder_and_upcast_attn': False, 'bos_token_id': 50256, 'eos_token_id': 50256, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': None, 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': True, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['GPT2LMHeadModel'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'pad_token_id': None, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': {'text-generation': {'do_sample': True, 'max_length': 50}}, 'problem_type': None, '_name_or_path': 'gpt2', 'transformers_version': '4.37.2', 'model_type': 'gpt2', 'n_ctx': 1024, 'model_class': <class 'transformers.models.gpt2.modeling_gpt2.GPT2LMHeadModel'>, 'hidden_size': 768}\n"
     ]
    }
   ],
   "source": [
    "model_params = get_model_params(model_path)\n",
    "model_class = model_params[\"model_class\"]\n",
    "hidden_size = model_params[\"hidden_size\"]\n",
    "vocab_size = model_params[\"vocab_size\"]\n",
    "print(model_params)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a3c60716",
   "metadata": {
    "papermill": {
     "duration": 0.003094,
     "end_time": "2024-03-23T23:23:39.224740",
     "exception": false,
     "start_time": "2024-03-23T23:23:39.221646",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "We are doing text classification, so we have to set pred_for_sequence to True for this task. In the imdb dataset, we only have two labels, 0 for negative and 1 for positive. So we have to set num_outputs to 2."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d37267c8-b10b-479b-b808-a7e5ffead033",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-23T23:23:39.231814Z",
     "iopub.status.busy": "2024-03-23T23:23:39.231584Z",
     "iopub.status.idle": "2024-03-23T23:23:39.235013Z",
     "shell.execute_reply": "2024-03-23T23:23:39.234551Z"
    },
    "papermill": {
     "duration": 0.008169,
     "end_time": "2024-03-23T23:23:39.236068",
     "exception": false,
     "start_time": "2024-03-23T23:23:39.227899",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "head_configs = [\n",
    "    HeadConfig(\n",
    "        name=f\"imdb_head_{(1+(i-1)*2)}\",\n",
    "        layer_hook=-(1 + (i - 1) * 2),\n",
    "        in_size=hidden_size,\n",
    "        output_activation=\"linear\",\n",
    "        pred_for_sequence=True,\n",
    "        loss_fct=\"cross_entropy\",\n",
    "        num_outputs=2,\n",
    "    )\n",
    "    for i in range(1, num_heads + 2)\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "487ed6ab-0386-4ab8-8cf5-e896952303f7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-23T23:23:39.244220Z",
     "iopub.status.busy": "2024-03-23T23:23:39.243819Z",
     "iopub.status.idle": "2024-03-23T23:23:46.537676Z",
     "shell.execute_reply": "2024-03-23T23:23:46.537198Z"
    },
    "papermill": {
     "duration": 7.299317,
     "end_time": "2024-03-23T23:23:46.539018",
     "exception": false,
     "start_time": "2024-03-23T23:23:39.239701",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "dd = load_dataset(\"imdb\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "58cccf74-e147-4612-afd9-7fadaea8d7f8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-23T23:23:46.546839Z",
     "iopub.status.busy": "2024-03-23T23:23:46.546414Z",
     "iopub.status.idle": "2024-03-23T23:23:46.552669Z",
     "shell.execute_reply": "2024-03-23T23:23:46.552224Z"
    },
    "papermill": {
     "duration": 0.011271,
     "end_time": "2024-03-23T23:23:46.553841",
     "exception": false,
     "start_time": "2024-03-23T23:23:46.542570",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'text': 'I love sci-fi and am willing to put up with a lot. Sci-fi movies/TV are usually underfunded, under-appreciated and misunderstood. I tried to like this, I really did, but it is to good TV sci-fi as Babylon 5 is to Star Trek (the original). Silly prosthetics, cheap cardboard sets, stilted dialogues, CG that doesn\\'t match the background, and painfully one-dimensional characters cannot be overcome with a \\'sci-fi\\' setting. (I\\'m sure there are those of you out there who think Babylon 5 is good sci-fi TV. It\\'s not. It\\'s clich\u00e9d and uninspiring.) While US viewers might like emotion and character development, sci-fi is a genre that does not take itself seriously (cf. Star Trek). It may treat important issues, yet not as a serious philosophy. It\\'s really difficult to care about the characters here as they are not simply foolish, just missing a spark of life. Their actions and reactions are wooden and predictable, often painful to watch. The makers of Earth KNOW it\\'s rubbish as they have to always say \"Gene Roddenberry\\'s Earth...\" otherwise people would not continue watching. Roddenberry\\'s ashes must be turning in their orbit as this dull, cheap, poorly edited (watching it without advert breaks really brings this home) trudging Trabant of a show lumbers into space. Spoiler. So, kill off a main character. And then bring him back as another actor. Jeeez! Dallas all over again.',\n",
       " 'label': 0}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dd[\"test\"][0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "96bdc467",
   "metadata": {
    "papermill": {
     "duration": 0.003503,
     "end_time": "2024-03-23T23:23:46.560986",
     "exception": false,
     "start_time": "2024-03-23T23:23:46.557483",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "In the *tokenize_function*, we set the *label* entry in the dataset for each of our heads."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "2ea1074e-b0b4-46b0-bc8f-dbb513a0386f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-23T23:23:46.568711Z",
     "iopub.status.busy": "2024-03-23T23:23:46.568315Z",
     "iopub.status.idle": "2024-03-23T23:24:07.638457Z",
     "shell.execute_reply": "2024-03-23T23:24:07.637799Z"
    },
    "papermill": {
     "duration": 21.075512,
     "end_time": "2024-03-23T23:24:07.639864",
     "exception": false,
     "start_time": "2024-03-23T23:23:46.564352",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "29edd71588d94046abd88b60ab71ef4d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/25000 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "416be6dd671f4cc68fc42e93ce6d9dbe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/25000 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "54ad7d7c6b8f4f2cbdd770dc0a9b5152",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/50000 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(model_path)\n",
    "if tokenizer.pad_token_id is None:\n",
    "    tokenizer.pad_token = tokenizer.eos_token\n",
    "\n",
    "\n",
    "def tokenize_function(examples):\n",
    "    out = tokenizer(examples[\"text\"], padding=False, truncation=True)\n",
    "    for hc in head_configs:\n",
    "        out[hc.name] = examples[\"label\"]\n",
    "    return out\n",
    "\n",
    "\n",
    "for split in dd.keys():\n",
    "    dd[split] = dd[split].filter(function=lambda example: len(example[\"text\"]) > 10)\n",
    "    dd[split] = dd[split].shuffle()\n",
    "    dd[split] = dd[split].map(tokenize_function, batched=True)\n",
    "\n",
    "dd.set_format(\n",
    "    type=\"torch\",\n",
    "    columns=[\"input_ids\", \"attention_mask\"] + [x.name for x in head_configs],\n",
    ")\n",
    "for split in dd.keys():\n",
    "    dd[split] = dd[split].remove_columns([\"text\", \"label\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8ea0bde2-594c-4b3c-bd8e-fd08777c10dd",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-23T23:24:07.650398Z",
     "iopub.status.busy": "2024-03-23T23:24:07.650107Z",
     "iopub.status.idle": "2024-03-23T23:24:09.148354Z",
     "shell.execute_reply": "2024-03-23T23:24:09.147842Z"
    },
    "papermill": {
     "duration": 1.504736,
     "end_time": "2024-03-23T23:24:09.149798",
     "exception": false,
     "start_time": "2024-03-23T23:24:07.645062",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of TransformerWithHeads were not initialized from the model checkpoint at gpt2 and are newly initialized: ['heads.imdb_head_1.lins.0.weight', 'heads.imdb_head_11.lins.0.weight', 'heads.imdb_head_13.lins.0.weight', 'heads.imdb_head_3.lins.0.weight', 'heads.imdb_head_5.lins.0.weight', 'heads.imdb_head_7.lins.0.weight', 'heads.imdb_head_9.lins.0.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "quantization_config = BitsAndBytesConfig(\n",
    "    load_in_4bit=True,\n",
    "    load_in_8bit=False,\n",
    "    llm_int8_threshold=6.0,\n",
    "    llm_int8_has_fp16_weight=False,\n",
    "    bnb_4bit_compute_dtype=torch.float32,\n",
    "    bnb_4bit_use_double_quant=True,\n",
    "    bnb_4bit_quant_type=\"nf4\",\n",
    ")\n",
    "\n",
    "model = load_headed(\n",
    "    model_class,\n",
    "    model_path,\n",
    "    head_configs=head_configs,\n",
    "    quantization_config=None if full_finetune else quantization_config,\n",
    "    freeze_base_model=not full_finetune,\n",
    "    device_map={\"\": torch.cuda.current_device()},\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "828af630-a734-4d0e-9d13-e86bf5f66cdb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-23T23:24:09.159040Z",
     "iopub.status.busy": "2024-03-23T23:24:09.158807Z",
     "iopub.status.idle": "2024-03-23T23:24:09.162538Z",
     "shell.execute_reply": "2024-03-23T23:24:09.161981Z"
    },
    "papermill": {
     "duration": 0.009625,
     "end_time": "2024-03-23T23:24:09.163832",
     "exception": false,
     "start_time": "2024-03-23T23:24:09.154207",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all params: 81983232 || trainable params: 10752 || trainable%: 0.01311487695435086\n",
      "params by dtype: defaultdict(<class 'int'>, {torch.float32: 39515904, torch.uint8: 42467328})\n",
      "trainable params by dtype: defaultdict(<class 'int'>, {torch.float32: 10752})\n"
     ]
    }
   ],
   "source": [
    "print_trainable_parameters(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8723801c",
   "metadata": {
    "papermill": {
     "duration": 0.003888,
     "end_time": "2024-03-23T23:24:09.171845",
     "exception": false,
     "start_time": "2024-03-23T23:24:09.167957",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Our heads are linear layers with only two outputs. Thus we have a very low amount of trainable parameters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "e3c107a0-6c81-4d08-af0f-ef690e6fc92e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-23T23:24:09.180503Z",
     "iopub.status.busy": "2024-03-23T23:24:09.180257Z",
     "iopub.status.idle": "2024-03-23T23:24:09.604295Z",
     "shell.execute_reply": "2024-03-23T23:24:09.603787Z"
    },
    "papermill": {
     "duration": 0.429748,
     "end_time": "2024-03-23T23:24:09.605539",
     "exception": false,
     "start_time": "2024-03-23T23:24:09.175791",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r\n",
      "Predicting:   0%|          | 0/5 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r\n",
      "Predicting:  20%|\u2588\u2588        | 1/5 [00:00<00:01,  3.84it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r\n",
      "Predicting: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 5/5 [00:00<00:00, 13.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                              review  label  ground_truth\n",
      "0  This movie is a waste of time. Though it has a...      0             0\n",
      "1  I saw this movie when it aired on Lifetime bac...      0             1\n",
      "2  Leslie Nielsen hits rock bottom with this abso...      0             0\n",
      "3  This BRASS EYE special PAEDO-GEDDON was swampe...      0             1\n",
      "4  You can't help but marvel at Hitchcock's early...      0             1\n",
      "5  i expected this movie to be absolutely god awf...      0             0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "ins, preds, ground_truths = get_some_preds(\n",
    "    model, dd[\"test\"], tokenizer, n=5, classification=True\n",
    ")\n",
    "print(\n",
    "    pd.DataFrame(\n",
    "        list(zip(ins, preds[\"imdb_head_5\"], ground_truths[\"imdb_head_5\"])),\n",
    "        columns=[\"review\", \"label\", \"ground_truth\"],\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "279113e4",
   "metadata": {
    "papermill": {
     "duration": 0.004309,
     "end_time": "2024-03-23T23:24:09.614417",
     "exception": false,
     "start_time": "2024-03-23T23:24:09.610108",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Untrained heads give fairly random outputs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "58359743-0cf4-43cd-97ae-48862d563b89",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-23T23:24:09.624061Z",
     "iopub.status.busy": "2024-03-23T23:24:09.623692Z",
     "iopub.status.idle": "2024-03-23T23:24:09.626576Z",
     "shell.execute_reply": "2024-03-23T23:24:09.626111Z"
    },
    "papermill": {
     "duration": 0.008866,
     "end_time": "2024-03-23T23:24:09.627636",
     "exception": false,
     "start_time": "2024-03-23T23:24:09.618770",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "collator = DataCollatorWithPadding(\n",
    "    feature_name_to_padding_value={\n",
    "        \"input_ids\": tokenizer.pad_token_id,\n",
    "        \"attention_mask\": 0,\n",
    "    }\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "193c099f-28be-40e6-ba93-22f326f6cd83",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-23T23:24:09.637609Z",
     "iopub.status.busy": "2024-03-23T23:24:09.637321Z",
     "iopub.status.idle": "2024-03-23T23:29:30.680838Z",
     "shell.execute_reply": "2024-03-23T23:29:30.680178Z"
    },
    "papermill": {
     "duration": 321.049603,
     "end_time": "2024-03-23T23:29:30.681951",
     "exception": false,
     "start_time": "2024-03-23T23:24:09.632348",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r\n",
      "Evaluating: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 3125/3125 [05:21<00:00,  9.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(12.376125398864746, {'imdb_head_1': 2.2763312496745587, 'imdb_head_3': 2.3986372451511024, 'imdb_head_5': 2.028720821541548, 'imdb_head_7': 1.2156276172590257, 'imdb_head_9': 1.8285838321861625, 'imdb_head_11': 1.9336576251229645, 'imdb_head_13': 0.694567006111145})\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "print(evaluate_head_wise(model, dd[\"test\"], collator, epochs=eval_epochs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "ac8e4faa-5e10-44af-a201-121565372a63",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-23T23:29:30.841937Z",
     "iopub.status.busy": "2024-03-23T23:29:30.841639Z",
     "iopub.status.idle": "2024-03-23T23:48:14.638565Z",
     "shell.execute_reply": "2024-03-23T23:48:14.637951Z"
    },
    "papermill": {
     "duration": 1123.878023,
     "end_time": "2024-03-23T23:48:14.639688",
     "exception": false,
     "start_time": "2024-03-23T23:29:30.761665",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mykeller\u001b[0m (\u001b[33mchm-hci\u001b[0m). Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: wandb version 0.16.4 is available!  To upgrade, please run:\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m:  $ pip install wandb --upgrade\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.16.3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m/raven/u/ykeller/transformer_heads/wandb/run-20240324_002932-elgaoj7t\u001b[0m\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33msplendid-spaceship-212\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u2b50\ufe0f View project at \u001b[34m\u001b[4mhttps://wandb.ai/chm-hci/huggingface\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: \ud83d\ude80 View run at \u001b[34m\u001b[4mhttps://wandb.ai/chm-hci/huggingface/runs/elgaoj7t\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/u/ykeller/conda-envs/sh_finetuning/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 18:37, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>100</td>\n",
       "      <td>5.814900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>200</td>\n",
       "      <td>5.118700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>300</td>\n",
       "      <td>5.112800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>400</td>\n",
       "      <td>4.929900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>4.847300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>600</td>\n",
       "      <td>4.806300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>700</td>\n",
       "      <td>4.718300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>800</td>\n",
       "      <td>4.814100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>900</td>\n",
       "      <td>4.702000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>4.703500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1100</td>\n",
       "      <td>4.671700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1200</td>\n",
       "      <td>4.598800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1300</td>\n",
       "      <td>4.585900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1400</td>\n",
       "      <td>4.531300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>4.535300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1600</td>\n",
       "      <td>4.499200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1700</td>\n",
       "      <td>4.524200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1800</td>\n",
       "      <td>4.487600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1900</td>\n",
       "      <td>4.376900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>4.451700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2100</td>\n",
       "      <td>4.463900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2200</td>\n",
       "      <td>4.363800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2300</td>\n",
       "      <td>4.372100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2400</td>\n",
       "      <td>4.390500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>4.373700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2600</td>\n",
       "      <td>4.256000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2700</td>\n",
       "      <td>4.364100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2800</td>\n",
       "      <td>4.329300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2900</td>\n",
       "      <td>4.287800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>4.382200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3100</td>\n",
       "      <td>4.289600</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Checkpoint destination directory imdb_linear_probe/checkpoint-500 already exists and is non-empty.Saving will proceed but saved results may be invalid.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/u/ykeller/conda-envs/sh_finetuning/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Checkpoint destination directory imdb_linear_probe/checkpoint-1000 already exists and is non-empty.Saving will proceed but saved results may be invalid.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/u/ykeller/conda-envs/sh_finetuning/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Checkpoint destination directory imdb_linear_probe/checkpoint-1500 already exists and is non-empty.Saving will proceed but saved results may be invalid.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/u/ykeller/conda-envs/sh_finetuning/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Checkpoint destination directory imdb_linear_probe/checkpoint-2000 already exists and is non-empty.Saving will proceed but saved results may be invalid.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/u/ykeller/conda-envs/sh_finetuning/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Checkpoint destination directory imdb_linear_probe/checkpoint-2500 already exists and is non-empty.Saving will proceed but saved results may be invalid.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/u/ykeller/conda-envs/sh_finetuning/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Checkpoint destination directory imdb_linear_probe/checkpoint-3000 already exists and is non-empty.Saving will proceed but saved results may be invalid.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/u/ykeller/conda-envs/sh_finetuning/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=3125, training_loss=4.601652141113282, metrics={'train_runtime': 1123.3071, 'train_samples_per_second': 22.256, 'train_steps_per_second': 2.782, 'total_flos': 8513909276590080.0, 'train_loss': 4.601652141113282, 'epoch': 1.0})"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "args = TrainingArguments(\n",
    "    output_dir=\"imdb_linear_probe\",\n",
    "    learning_rate=0.0002,\n",
    "    num_train_epochs=train_epochs,  # To speed things up set to 0.1, set to 1 for better performance\n",
    "    logging_steps=logging_steps,\n",
    "    do_eval=False,\n",
    "    remove_unused_columns=False,\n",
    ")\n",
    "trainer = Trainer(\n",
    "    model,\n",
    "    args=args,\n",
    "    train_dataset=dd[\"train\"],\n",
    "    data_collator=collator,\n",
    ")\n",
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "8c239044-aef1-4509-a723-819a3a107928",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-23T23:48:14.807804Z",
     "iopub.status.busy": "2024-03-23T23:48:14.807518Z",
     "iopub.status.idle": "2024-03-23T23:53:51.527107Z",
     "shell.execute_reply": "2024-03-23T23:53:51.526520Z"
    },
    "papermill": {
     "duration": 336.803787,
     "end_time": "2024-03-23T23:53:51.528235",
     "exception": false,
     "start_time": "2024-03-23T23:48:14.724448",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r\n",
      "Evaluating: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 3125/3125 [05:36<00:00,  9.28it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/u/ykeller/conda-envs/sh_finetuning/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "\r\n",
      "Evaluating:   0%|          | 2/3125 [00:00<05:12, 10.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4.350820428314209, {'imdb_head_1': 0.6196392296218872, 'imdb_head_3': 0.5805930749511719, 'imdb_head_5': 0.5834007847213745, 'imdb_head_7': 0.6084902699661255, 'imdb_head_9': 0.624310562505722, 'imdb_head_11': 0.6407481147956848, 'imdb_head_13': 0.6936383946609497})\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "print(evaluate_head_wise(model, dd[\"test\"], collator, epochs=eval_epochs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "38a9a41a-d8a5-49d3-8e7c-f81bfbe730df",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-23T23:53:51.854389Z",
     "iopub.status.busy": "2024-03-23T23:53:51.853680Z",
     "iopub.status.idle": "2024-03-23T23:53:52.024300Z",
     "shell.execute_reply": "2024-03-23T23:53:52.023821Z"
    },
    "papermill": {
     "duration": 0.334853,
     "end_time": "2024-03-23T23:53:52.025496",
     "exception": false,
     "start_time": "2024-03-23T23:53:51.690643",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r\n",
      "Predicting:   0%|          | 0/5 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r\n",
      "Predicting:  80%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588  | 4/5 [00:00<00:00, 37.13it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r\n",
      "Predicting: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 5/5 [00:00<00:00, 31.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                              review  label  ground_truth\n",
      "0  This movie is a waste of time. Though it has a...      0             0\n",
      "1  I saw this movie when it aired on Lifetime bac...      1             1\n",
      "2  Leslie Nielsen hits rock bottom with this abso...      0             0\n",
      "3  This BRASS EYE special PAEDO-GEDDON was swampe...      0             1\n",
      "4  You can't help but marvel at Hitchcock's early...      1             1\n",
      "5  i expected this movie to be absolutely god awf...      0             0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "ins, preds, ground_truths = get_some_preds(\n",
    "    model, dd[\"test\"], tokenizer, n=5, classification=True\n",
    ")\n",
    "print(\n",
    "    pd.DataFrame(\n",
    "        list(zip(ins, preds[\"imdb_head_5\"], ground_truths[\"imdb_head_5\"])),\n",
    "        columns=[\"review\", \"label\", \"ground_truth\"],\n",
    "    )\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "sh_finetuning",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 1822.77361,
   "end_time": "2024-03-23T23:53:54.931862",
   "environment_variables": {},
   "exception": null,
   "input_path": "notebooks/base/text_classification.ipynb",
   "output_path": "notebooks/gpt2/text_classification_linear_probe.ipynb",
   "parameters": {
    "eval_epochs": 1,
    "full_finetune": false,
    "logging_steps": 100,
    "model_path": "gpt2",
    "num_heads": 6,
    "train_epochs": 1
   },
   "start_time": "2024-03-23T23:23:32.158252",
   "version": "2.5.0"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "059d0f1c34c741eb96b932d8e9d19d15": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_432da154d9324a23b4a528533db3b815",
       "max": 50000,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_4b8105a4a1c34bafa6a0c13e2a9751f9",
       "tabbable": null,
       "tooltip": null,
       "value": 50000
      }
     },
     "1b68b1dbf3da4b9692fa825dd92ffb12": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "2871a514c6a54655b4203c48fef546cc": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "29edd71588d94046abd88b60ab71ef4d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_d4212203edd144d99b00e9e0394690c2",
        "IPY_MODEL_76528bd0ccc941b6a31e0cfeaae46019",
        "IPY_MODEL_32dbfef0e5f341b29afcf116e1dab4bf"
       ],
       "layout": "IPY_MODEL_e2d6f1c3882b42df8e225a3dc41a6507",
       "tabbable": null,
       "tooltip": null
      }
     },
     "2bfdf733700b43a4a6a2cfc115b9e538": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "2c1bfb4e167f499fbd5700be6199b63e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "32dbfef0e5f341b29afcf116e1dab4bf": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_ccf7f1ec0cb3495ba94575dbf047b59c",
       "placeholder": "\u200b",
       "style": "IPY_MODEL_2bfdf733700b43a4a6a2cfc115b9e538",
       "tabbable": null,
       "tooltip": null,
       "value": "\u200725000/25000\u2007[00:05&lt;00:00,\u20074972.84\u2007examples/s]"
      }
     },
     "38769ce629024930bd4b234ae9271082": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "3bf5a22b4c7c4c46b6f5ec58b1308a12": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "416be6dd671f4cc68fc42e93ce6d9dbe": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_f8d7f4d75df54c2daf79f1343643e6fd",
        "IPY_MODEL_a7a3ec2d0791403ab1a0f4568ddc531f",
        "IPY_MODEL_ac61df580fb743928e0994e296cde530"
       ],
       "layout": "IPY_MODEL_cf84f2da15fa4d268397f682fd99294b",
       "tabbable": null,
       "tooltip": null
      }
     },
     "432da154d9324a23b4a528533db3b815": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "4984f246310f42e78930c6021292a171": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "4b8105a4a1c34bafa6a0c13e2a9751f9": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "52e787181033437b883fd230a9567e63": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "54ad7d7c6b8f4f2cbdd770dc0a9b5152": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_758fc0af35d74184ae667f0526da089b",
        "IPY_MODEL_059d0f1c34c741eb96b932d8e9d19d15",
        "IPY_MODEL_8a2cc7dd997c49b4ae6541b0fdda3b02"
       ],
       "layout": "IPY_MODEL_3bf5a22b4c7c4c46b6f5ec58b1308a12",
       "tabbable": null,
       "tooltip": null
      }
     },
     "610086690f1f4050ad87291d27b10e25": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "758fc0af35d74184ae667f0526da089b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_e8f5fc70dfb24bd19ebd9beb1fc6595c",
       "placeholder": "\u200b",
       "style": "IPY_MODEL_bd30e402a1d94d97a684ab087376f49c",
       "tabbable": null,
       "tooltip": null,
       "value": "Map:\u2007100%"
      }
     },
     "76528bd0ccc941b6a31e0cfeaae46019": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_4984f246310f42e78930c6021292a171",
       "max": 25000,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_610086690f1f4050ad87291d27b10e25",
       "tabbable": null,
       "tooltip": null,
       "value": 25000
      }
     },
     "7d4da04a5cf54c0ba41129a7dc1c74cf": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "8a2cc7dd997c49b4ae6541b0fdda3b02": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_1b68b1dbf3da4b9692fa825dd92ffb12",
       "placeholder": "\u200b",
       "style": "IPY_MODEL_2c1bfb4e167f499fbd5700be6199b63e",
       "tabbable": null,
       "tooltip": null,
       "value": "\u200750000/50000\u2007[00:10&lt;00:00,\u20074817.25\u2007examples/s]"
      }
     },
     "a134d0ca47c24c388396aa92cb596ef9": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "a7a3ec2d0791403ab1a0f4568ddc531f": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_e8709f6d5392491aae97f9fee6cee26e",
       "max": 25000,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_a134d0ca47c24c388396aa92cb596ef9",
       "tabbable": null,
       "tooltip": null,
       "value": 25000
      }
     },
     "ac61df580fb743928e0994e296cde530": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_d9452f2e4f37403f8c9ff5dc8e89a4d3",
       "placeholder": "\u200b",
       "style": "IPY_MODEL_7d4da04a5cf54c0ba41129a7dc1c74cf",
       "tabbable": null,
       "tooltip": null,
       "value": "\u200725000/25000\u2007[00:04&lt;00:00,\u20075177.55\u2007examples/s]"
      }
     },
     "bd30e402a1d94d97a684ab087376f49c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "ccf7f1ec0cb3495ba94575dbf047b59c": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "cf84f2da15fa4d268397f682fd99294b": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "d4212203edd144d99b00e9e0394690c2": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_2871a514c6a54655b4203c48fef546cc",
       "placeholder": "\u200b",
       "style": "IPY_MODEL_38769ce629024930bd4b234ae9271082",
       "tabbable": null,
       "tooltip": null,
       "value": "Map:\u2007100%"
      }
     },
     "d9452f2e4f37403f8c9ff5dc8e89a4d3": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "e2d6f1c3882b42df8e225a3dc41a6507": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "e8709f6d5392491aae97f9fee6cee26e": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "e8f5fc70dfb24bd19ebd9beb1fc6595c": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "f5c17bbe6815441692c3627aab15b080": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "f8d7f4d75df54c2daf79f1343643e6fd": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_52e787181033437b883fd230a9567e63",
       "placeholder": "\u200b",
       "style": "IPY_MODEL_f5c17bbe6815441692c3627aab15b080",
       "tabbable": null,
       "tooltip": null,
       "value": "Map:\u2007100%"
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}