blender_bot_pruning_strat.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "75ec25fe-bed3-43a9-94e8-22413302a7a0",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/homebrew/lib/python3.10/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import numpy as np\n",
    "import pickle as pkl\n",
    "\n",
    "from node_attribution.blender_bot_gradient_node_attribution import get_attributions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "dd5f02b6-a727-47ed-baf0-14ae880caa13",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "human_filtered_pairs = pkl.load(open(\"44_human_filtered_conv_pairs.pkl\", \"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f9cc4ca1-1604-4734-925b-39eed900da8e",
   "metadata": {},
   "outputs": [],
   "source": [
    "calibration_data = human_filtered_pairs[:22]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "8efbe0a3-048c-4ca6-a77e-dfe9315161ad",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading (…)okenizer_config.json: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████| 74.0/74.0 [00:00<00:00, 25.1kB/s]\n",
      "Downloading (…)lve/main/config.json: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████| 1.29k/1.29k [00:00<00:00, 471kB/s]\n",
      "Downloading (…)olve/main/vocab.json: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████| 150k/150k [00:00<00:00, 952kB/s]\n",
      "Downloading (…)olve/main/merges.txt: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████| 62.9k/62.9k [00:00<00:00, 423kB/s]\n",
      "Downloading (…)cial_tokens_map.json: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████| 130/130 [00:00<00:00, 15.0kB/s]\n",
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
      "Downloading (…)\"pytorch_model.bin\";: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████| 5.47G/5.47G [05:37<00:00, 16.2MB/s]\n",
      "Downloading (…)neration_config.json: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████| 347/347 [00:00<00:00, 70.8kB/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished loading facebook/blenderbot-3B.\n",
      "                    Num Encoder Transformer Blocks: 2\n",
      "                    Num Decoder Transformer Blocks: 24\n",
      "                    Num Encoder Attention Heads: 32\n",
      "                    Num Decoder Attention Heads: 32\n",
      "                    Encoder Head Dim: 80\n",
      "                    Decoder Head Dim: 80\n",
      "                    Hidden Size: 2560\n",
      "                    Base Model Param Count: 2,696,268,800\n",
      "                    Total Param Count (w/ LM Head): 2,716,769,280\n",
      "final_logits_bias torch.Size([1, 8008])\n",
      "model.shared.weight torch.Size([8008, 2560])\n",
      "model.encoder.embed_tokens.weight torch.Size([8008, 2560])\n",
      "model.encoder.embed_positions.weight torch.Size([128, 2560])\n",
      "model.encoder.layers.0.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.encoder.layers.0.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.encoder.layers.0.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.encoder.layers.0.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.encoder.layers.0.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.encoder.layers.0.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.encoder.layers.0.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.encoder.layers.0.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.encoder.layers.0.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.encoder.layers.0.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.encoder.layers.0.fc1.weight torch.Size([10240, 2560])\n",
      "model.encoder.layers.0.fc1.bias torch.Size([10240])\n",
      "model.encoder.layers.0.fc2.weight torch.Size([2560, 10240])\n",
      "model.encoder.layers.0.fc2.bias torch.Size([2560])\n",
      "model.encoder.layers.0.final_layer_norm.weight torch.Size([2560])\n",
      "model.encoder.layers.0.final_layer_norm.bias torch.Size([2560])\n",
      "model.encoder.layers.1.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.encoder.layers.1.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.encoder.layers.1.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.encoder.layers.1.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.encoder.layers.1.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.encoder.layers.1.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.encoder.layers.1.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.encoder.layers.1.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.encoder.layers.1.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.encoder.layers.1.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.encoder.layers.1.fc1.weight torch.Size([10240, 2560])\n",
      "model.encoder.layers.1.fc1.bias torch.Size([10240])\n",
      "model.encoder.layers.1.fc2.weight torch.Size([2560, 10240])\n",
      "model.encoder.layers.1.fc2.bias torch.Size([2560])\n",
      "model.encoder.layers.1.final_layer_norm.weight torch.Size([2560])\n",
      "model.encoder.layers.1.final_layer_norm.bias torch.Size([2560])\n",
      "model.encoder.layer_norm.weight torch.Size([2560])\n",
      "model.encoder.layer_norm.bias torch.Size([2560])\n",
      "model.decoder.embed_tokens.weight torch.Size([8008, 2560])\n",
      "model.decoder.embed_positions.weight torch.Size([128, 2560])\n",
      "model.decoder.layers.0.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.0.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.0.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.0.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.0.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.0.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.0.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.0.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.0.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.0.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.0.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.0.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.0.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.0.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.0.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.0.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.0.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.0.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.0.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.0.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.0.fc1.weight torch.Size([10240, 2560])\n",
      "model.decoder.layers.0.fc1.bias torch.Size([10240])\n",
      "model.decoder.layers.0.fc2.weight torch.Size([2560, 10240])\n",
      "model.decoder.layers.0.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.0.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.0.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.1.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.1.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.1.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.1.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.1.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.1.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.1.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.1.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.1.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.1.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.1.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.1.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.1.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.1.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.1.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.1.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.1.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.1.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.1.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.1.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.1.fc1.weight torch.Size([10240, 2560])\n",
      "model.decoder.layers.1.fc1.bias torch.Size([10240])\n",
      "model.decoder.layers.1.fc2.weight torch.Size([2560, 10240])\n",
      "model.decoder.layers.1.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.1.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.1.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.2.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.2.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.2.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.2.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.2.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.2.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.2.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.2.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.2.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.2.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.2.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.2.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.2.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.2.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.2.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.2.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.2.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.2.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.2.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.2.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.2.fc1.weight torch.Size([10240, 2560])\n",
      "model.decoder.layers.2.fc1.bias torch.Size([10240])\n",
      "model.decoder.layers.2.fc2.weight torch.Size([2560, 10240])\n",
      "model.decoder.layers.2.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.2.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.2.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.3.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.3.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.3.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.3.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.3.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.3.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.3.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.3.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.3.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.3.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.3.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.3.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.3.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.3.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.3.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.3.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.3.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.3.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.3.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.3.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.3.fc1.weight torch.Size([10240, 2560])\n",
      "model.decoder.layers.3.fc1.bias torch.Size([10240])\n",
      "model.decoder.layers.3.fc2.weight torch.Size([2560, 10240])\n",
      "model.decoder.layers.3.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.3.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.3.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.4.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.4.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.4.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.4.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.4.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.4.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.4.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.4.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.4.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.4.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.4.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.4.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.4.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.4.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.4.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.4.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.4.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.4.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.4.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.4.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.4.fc1.weight torch.Size([10240, 2560])\n",
      "model.decoder.layers.4.fc1.bias torch.Size([10240])\n",
      "model.decoder.layers.4.fc2.weight torch.Size([2560, 10240])\n",
      "model.decoder.layers.4.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.4.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.4.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.5.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.5.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.5.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.5.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.5.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.5.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.5.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.5.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.5.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.5.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.5.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.5.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.5.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.5.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.5.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.5.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.5.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.5.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.5.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.5.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.5.fc1.weight torch.Size([10240, 2560])\n",
      "model.decoder.layers.5.fc1.bias torch.Size([10240])\n",
      "model.decoder.layers.5.fc2.weight torch.Size([2560, 10240])\n",
      "model.decoder.layers.5.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.5.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.5.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.6.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.6.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.6.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.6.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.6.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.6.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.6.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.6.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.6.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.6.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.6.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.6.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.6.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.6.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.6.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.6.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.6.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.6.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.6.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.6.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.6.fc1.weight torch.Size([10240, 2560])\n",
      "model.decoder.layers.6.fc1.bias torch.Size([10240])\n",
      "model.decoder.layers.6.fc2.weight torch.Size([2560, 10240])\n",
      "model.decoder.layers.6.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.6.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.6.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.7.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.7.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.7.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.7.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.7.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.7.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.7.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.7.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.7.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.7.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.7.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.7.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.7.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.7.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.7.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.7.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.7.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.7.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.7.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.7.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.7.fc1.weight torch.Size([10240, 2560])\n",
      "model.decoder.layers.7.fc1.bias torch.Size([10240])\n",
      "model.decoder.layers.7.fc2.weight torch.Size([2560, 10240])\n",
      "model.decoder.layers.7.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.7.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.7.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.8.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.8.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.8.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.8.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.8.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.8.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.8.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.8.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.8.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.8.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.8.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.8.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.8.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.8.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.8.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.8.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.8.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.8.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.8.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.8.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.8.fc1.weight torch.Size([10240, 2560])\n",
      "model.decoder.layers.8.fc1.bias torch.Size([10240])\n",
      "model.decoder.layers.8.fc2.weight torch.Size([2560, 10240])\n",
      "model.decoder.layers.8.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.8.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.8.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.9.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.9.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.9.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.9.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.9.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.9.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.9.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.9.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.9.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.9.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.9.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.9.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.9.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.9.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.9.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.9.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.9.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.9.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.9.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.9.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.9.fc1.weight torch.Size([10240, 2560])\n",
      "model.decoder.layers.9.fc1.bias torch.Size([10240])\n",
      "model.decoder.layers.9.fc2.weight torch.Size([2560, 10240])\n",
      "model.decoder.layers.9.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.9.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.9.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.10.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.10.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.10.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.10.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.10.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.10.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.10.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.10.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.10.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.10.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.10.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.10.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.10.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.10.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.10.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.10.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.10.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.10.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.10.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.10.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.10.fc1.weight torch.Size([10240, 2560])\n",
      "model.decoder.layers.10.fc1.bias torch.Size([10240])\n",
      "model.decoder.layers.10.fc2.weight torch.Size([2560, 10240])\n",
      "model.decoder.layers.10.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.10.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.10.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.11.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.11.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.11.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.11.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.11.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.11.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.11.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.11.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.11.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.11.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.11.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.11.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.11.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.11.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.11.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.11.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.11.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.11.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.11.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.11.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.11.fc1.weight torch.Size([10240, 2560])\n",
      "model.decoder.layers.11.fc1.bias torch.Size([10240])\n",
      "model.decoder.layers.11.fc2.weight torch.Size([2560, 10240])\n",
      "model.decoder.layers.11.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.11.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.11.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.12.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.12.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.12.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.12.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.12.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.12.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.12.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.12.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.12.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.12.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.12.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.12.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.12.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.12.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.12.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.12.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.12.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.12.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.12.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.12.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.12.fc1.weight torch.Size([10240, 2560])\n",
      "model.decoder.layers.12.fc1.bias torch.Size([10240])\n",
      "model.decoder.layers.12.fc2.weight torch.Size([2560, 10240])\n",
      "model.decoder.layers.12.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.12.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.12.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.13.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.13.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.13.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.13.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.13.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.13.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.13.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.13.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.13.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.13.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.13.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.13.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.13.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.13.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.13.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.13.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.13.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.13.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.13.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.13.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.13.fc1.weight torch.Size([10240, 2560])\n",
      "model.decoder.layers.13.fc1.bias torch.Size([10240])\n",
      "model.decoder.layers.13.fc2.weight torch.Size([2560, 10240])\n",
      "model.decoder.layers.13.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.13.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.13.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.14.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.14.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.14.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.14.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.14.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.14.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.14.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.14.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.14.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.14.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.14.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.14.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.14.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.14.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.14.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.14.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.14.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.14.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.14.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.14.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.14.fc1.weight torch.Size([10240, 2560])\n",
      "model.decoder.layers.14.fc1.bias torch.Size([10240])\n",
      "model.decoder.layers.14.fc2.weight torch.Size([2560, 10240])\n",
      "model.decoder.layers.14.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.14.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.14.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.15.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.15.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.15.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.15.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.15.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.15.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.15.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.15.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.15.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.15.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.15.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.15.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.15.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.15.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.15.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.15.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.15.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.15.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.15.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.15.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.15.fc1.weight torch.Size([10240, 2560])\n",
      "model.decoder.layers.15.fc1.bias torch.Size([10240])\n",
      "model.decoder.layers.15.fc2.weight torch.Size([2560, 10240])\n",
      "model.decoder.layers.15.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.15.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.15.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.16.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.16.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.16.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.16.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.16.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.16.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.16.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.16.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.16.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.16.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.16.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.16.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.16.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.16.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.16.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.16.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.16.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.16.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.16.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.16.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.16.fc1.weight torch.Size([10240, 2560])\n",
      "model.decoder.layers.16.fc1.bias torch.Size([10240])\n",
      "model.decoder.layers.16.fc2.weight torch.Size([2560, 10240])\n",
      "model.decoder.layers.16.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.16.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.16.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.17.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.17.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.17.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.17.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.17.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.17.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.17.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.17.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.17.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.17.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.17.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.17.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.17.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.17.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.17.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.17.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.17.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.17.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.17.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.17.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.17.fc1.weight torch.Size([10240, 2560])\n",
      "model.decoder.layers.17.fc1.bias torch.Size([10240])\n",
      "model.decoder.layers.17.fc2.weight torch.Size([2560, 10240])\n",
      "model.decoder.layers.17.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.17.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.17.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.18.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.18.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.18.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.18.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.18.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.18.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.18.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.18.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.18.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.18.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.18.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.18.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.18.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.18.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.18.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.18.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.18.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.18.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.18.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.18.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.18.fc1.weight torch.Size([10240, 2560])\n",
      "model.decoder.layers.18.fc1.bias torch.Size([10240])\n",
      "model.decoder.layers.18.fc2.weight torch.Size([2560, 10240])\n",
      "model.decoder.layers.18.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.18.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.18.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.19.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.19.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.19.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.19.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.19.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.19.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.19.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.19.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.19.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.19.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.19.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.19.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.19.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.19.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.19.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.19.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.19.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.19.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.19.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.19.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.19.fc1.weight torch.Size([10240, 2560])\n",
      "model.decoder.layers.19.fc1.bias torch.Size([10240])\n",
      "model.decoder.layers.19.fc2.weight torch.Size([2560, 10240])\n",
      "model.decoder.layers.19.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.19.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.19.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.20.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.20.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.20.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.20.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.20.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.20.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.20.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.20.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.20.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.20.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.20.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.20.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.20.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.20.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.20.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.20.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.20.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.20.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.20.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.20.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.20.fc1.weight torch.Size([10240, 2560])\n",
      "model.decoder.layers.20.fc1.bias torch.Size([10240])\n",
      "model.decoder.layers.20.fc2.weight torch.Size([2560, 10240])\n",
      "model.decoder.layers.20.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.20.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.20.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.21.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.21.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.21.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.21.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.21.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.21.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.21.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.21.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.21.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.21.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.21.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.21.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.21.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.21.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.21.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.21.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.21.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.21.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.21.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.21.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.21.fc1.weight torch.Size([10240, 2560])\n",
      "model.decoder.layers.21.fc1.bias torch.Size([10240])\n",
      "model.decoder.layers.21.fc2.weight torch.Size([2560, 10240])\n",
      "model.decoder.layers.21.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.21.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.21.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.22.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.22.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.22.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.22.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.22.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.22.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.22.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.22.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.22.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.22.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.22.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.22.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.22.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.22.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.22.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.22.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.22.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.22.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.22.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.22.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.22.fc1.weight torch.Size([10240, 2560])\n",
      "model.decoder.layers.22.fc1.bias torch.Size([10240])\n",
      "model.decoder.layers.22.fc2.weight torch.Size([2560, 10240])\n",
      "model.decoder.layers.22.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.22.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.22.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.23.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.23.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.23.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.23.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.23.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.23.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.23.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.23.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.23.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.23.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.23.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.23.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.23.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.23.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.23.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.23.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.23.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.23.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.23.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.23.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.23.fc1.weight torch.Size([10240, 2560])\n",
      "model.decoder.layers.23.fc1.bias torch.Size([10240])\n",
      "model.decoder.layers.23.fc2.weight torch.Size([2560, 10240])\n",
      "model.decoder.layers.23.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.23.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.23.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layer_norm.bias torch.Size([2560])\n",
      "lm_head.weight torch.Size([8008, 2560])\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n"
     ]
    }
   ],
   "source": [
    "# Get attributions\n",
    "model_size = \"3B\"\n",
    "avg_contributions, max_contributions, model, model_params = get_attributions(model_size, calibration_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "b9b7a08a-65d7-4158-9e60-fd6839f59c81",
   "metadata": {},
   "outputs": [],
   "source": [
    "# take absolute value of average contributions\n",
    "for layer_name, contribution_tensor in avg_contributions.items():\n",
    "    avg_contributions[layer_name] = torch.abs(avg_contributions[layer_name])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "7840688f-5c3f-4670-b280-0e7a26cef4ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "pkl.dump(avg_contributions, open(\"avg_contri_blender_bot_3B_22pair_calibration.pkl\", \"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "d57fcb4a-9f44-48d3-9267-5620a8dcfb25",
   "metadata": {},
   "outputs": [],
   "source": [
    "pkl.dump(max_contributions, open(\"max_contri_blender_bot_3B_22pair_calibration.pkl\", \"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "id": "289d2a47-0d93-43ad-897b-3ae05dfcce49",
   "metadata": {},
   "outputs": [],
   "source": [
    "prune_percent = 0.15\n",
    "num_params_to_prune = 2696268800 * prune_percent\n",
    "\n",
    "head_dim = 80\n",
    "num_heads = 32"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "id": "4e880a64-b350-4423-9298-5ef32f86976c",
   "metadata": {},
   "outputs": [],
   "source": [
    "index = 0\n",
    "params_to_index = {}\n",
    "for param_name in model_params.keys():\n",
    "    if \"bias\" not in param_name:\n",
    "        params_to_index[param_name] = index\n",
    "        index += 1\n",
    "    \n",
    "index_to_params = {params_to_index[param_name]: param_name for param_name in params_to_index.keys()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "id": "f993bcaa-4ce4-45bf-b076-63710af5d266",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_nodes = []\n",
    "\n",
    "for layer_name, contribution_tensor in avg_contributions.items():\n",
    "    if \"fc2.weight\" in layer_name:\n",
    "        for node_id, node in enumerate(contribution_tensor.tolist()):\n",
    "            node_name = f\"{layer_name}.{node_id}\"\n",
    "            all_nodes.append((node_name, node))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "id": "7061e785-c991-4e60-b4c3-80bfb778cf1c",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_layers = []\n",
    "\n",
    "# Figure out which layers have the most to prune\n",
    "for layer_name, contribution_tensor in avg_contributions.items():\n",
    "    \n",
    "    if \"fc2.weight\" in layer_name:\n",
    "        # Get average contribution over the whole layer\n",
    "        mean_contribution = torch.mean(contribution_tensor, 0).item()\n",
    "        all_layers.append((layer_name, mean_contribution))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "id": "6568baf2-3382-4e41-a309-e20b827d8830",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_layers.sort(key = lambda x:x[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "id": "c4c11859-bded-4210-8fbd-2a5cae586151",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('model.encoder.layers.0.fc2.weight', 1.6720172425266355e-05),\n",
       " ('model.encoder.layers.1.fc2.weight', 2.361466977163218e-05),\n",
       " ('model.decoder.layers.22.fc2.weight', 9.73643982433714e-05),\n",
       " ('model.decoder.layers.21.fc2.weight', 0.0001065301476046443),\n",
       " ('model.decoder.layers.20.fc2.weight', 0.00010903981456067413),\n",
       " ('model.decoder.layers.23.fc2.weight', 0.00010948643466690555),\n",
       " ('model.decoder.layers.19.fc2.weight', 0.00011663961049634963),\n",
       " ('model.decoder.layers.18.fc2.weight', 0.00012284422700759023),\n",
       " ('model.decoder.layers.17.fc2.weight', 0.00012652072473429143),\n",
       " ('model.decoder.layers.16.fc2.weight', 0.00013740375288762152),\n",
       " ('model.decoder.layers.15.fc2.weight', 0.0001586541038705036),\n",
       " ('model.decoder.layers.14.fc2.weight', 0.00016885095101315528),\n",
       " ('model.decoder.layers.13.fc2.weight', 0.00018126037321053445),\n",
       " ('model.decoder.layers.12.fc2.weight', 0.00018229440320283175),\n",
       " ('model.decoder.layers.11.fc2.weight', 0.00021381850820034742),\n",
       " ('model.decoder.layers.0.fc2.weight', 0.0002290532283950597),\n",
       " ('model.decoder.layers.1.fc2.weight', 0.00024232288706116378),\n",
       " ('model.decoder.layers.10.fc2.weight', 0.00024376981309615076),\n",
       " ('model.decoder.layers.2.fc2.weight', 0.0002551245561335236),\n",
       " ('model.decoder.layers.3.fc2.weight', 0.00026891581364907324),\n",
       " ('model.decoder.layers.8.fc2.weight', 0.0002819729852490127),\n",
       " ('model.decoder.layers.6.fc2.weight', 0.0002828227879945189),\n",
       " ('model.decoder.layers.4.fc2.weight', 0.00028568162815645337),\n",
       " ('model.decoder.layers.9.fc2.weight', 0.0002859591331798583),\n",
       " ('model.decoder.layers.5.fc2.weight', 0.0002902276464737952),\n",
       " ('model.decoder.layers.7.fc2.weight', 0.00029742586775682867)]"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "id": "b6f75b8e-7803-464c-b368-21c453ba0725",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "layer_masks = {}\n",
    "num_params_pruned = 0\n",
    "node_num = 0\n",
    "min_nodes = 24\n",
    "\n",
    "while num_params_pruned < num_params_to_prune:\n",
    "    lowest_contr_layer_name = all_layers[0][0]               \n",
    "    stop_pruning_layer = False\n",
    " \n",
    "    # Prune one node at time\n",
    "    if lowest_contr_layer_name not in layer_masks:\n",
    "        layer_contributions = avg_contributions[lowest_contr_layer_name]\n",
    "        mask = torch.zeros_like(layer_contributions)\n",
    "        sorted_contributions = torch.argsort(layer_contributions)\n",
    "        num_pruned = 0\n",
    "\n",
    "    else:\n",
    "        mask = layer_masks[lowest_contr_layer_name][0]\n",
    "        sorted_contributions = layer_masks[lowest_contr_layer_name][1]\n",
    "        num_pruned = layer_masks[lowest_contr_layer_name][2]\n",
    "\n",
    "    index_to_mask = sorted_contributions[num_pruned]\n",
    "    mask[index_to_mask] = 1\n",
    "\n",
    "    nodes_left = torch.numel(mask) - int(torch.sum(mask).item())\n",
    "\n",
    "    # Keep from deleting all nodes in a layer\n",
    "    if nodes_left > min_nodes:\n",
    "        layer_masks[lowest_contr_layer_name] = (mask, sorted_contributions, num_pruned + 1)\n",
    "        num_params_pruned += 5120\n",
    "        node_num += 1\n",
    "    else:\n",
    "        stop_pruning_layer = True\n",
    "\n",
    "    # Apply mask and update the layer mean in \"all_layers\"\n",
    "    if stop_pruning_layer:\n",
    "        new_layer_contr_score = float('inf')\n",
    "    else:\n",
    "        mean_array = np.ma.array(data=avg_contributions[lowest_contr_layer_name], mask=mask)\n",
    "        new_layer_contr_score = mean_array.mean()\n",
    "\n",
    "    all_layers[0] = (lowest_contr_layer_name, new_layer_contr_score)\n",
    "\n",
    "    # re-sort layers now that this one has been pruned and pick the lowest contributing layer again\n",
    "    all_layers.sort(key = lambda x:x[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "id": "0206985a-0ff8-4ce5-851d-51db90587410",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'model.encoder.layers.0.fc2.weight': (tensor([1., 1., 1.,  ..., 1., 1., 1.]),\n",
       "  tensor([5048, 8807, 1189,  ..., 3884, 5643, 4459]),\n",
       "  10215),\n",
       " 'model.encoder.layers.1.fc2.weight': (tensor([1., 1., 1.,  ..., 1., 1., 1.]),\n",
       "  tensor([2575, 4733, 6676,  ..., 9328, 7370, 6048]),\n",
       "  10150),\n",
       " 'model.decoder.layers.22.fc2.weight': (tensor([1., 1., 1.,  ..., 0., 0., 1.]),\n",
       "  tensor([ 740, 2624, 2448,  ..., 4023, 8360, 8561]),\n",
       "  7573),\n",
       " 'model.decoder.layers.21.fc2.weight': (tensor([0., 1., 1.,  ..., 1., 0., 0.]),\n",
       "  tensor([ 3977,  5433,  7186,  ...,  9161, 10129,  6500]),\n",
       "  7104),\n",
       " 'model.decoder.layers.20.fc2.weight': (tensor([0., 1., 1.,  ..., 1., 1., 1.]),\n",
       "  tensor([10016,  5858,  3353,  ...,  9204,  7077,  5807]),\n",
       "  6894),\n",
       " 'model.decoder.layers.23.fc2.weight': (tensor([1., 1., 0.,  ..., 1., 1., 0.]),\n",
       "  tensor([5087, 5559, 3759,  ..., 7899, 4355,  709]),\n",
       "  6362),\n",
       " 'model.decoder.layers.19.fc2.weight': (tensor([1., 0., 1.,  ..., 0., 1., 0.]),\n",
       "  tensor([3420, 3955, 8052,  ..., 2362,  234, 1810]),\n",
       "  6248),\n",
       " 'model.decoder.layers.18.fc2.weight': (tensor([1., 1., 1.,  ..., 1., 1., 0.]),\n",
       "  tensor([ 8255,  6986,  3927,  ...,  5864,  5226, 10206]),\n",
       "  5789),\n",
       " 'model.decoder.layers.17.fc2.weight': (tensor([1., 1., 0.,  ..., 0., 0., 1.]),\n",
       "  tensor([3439, 3523, 8938,  ..., 3178, 6932, 2116]),\n",
       "  5484),\n",
       " 'model.decoder.layers.16.fc2.weight': (tensor([1., 0., 1.,  ..., 1., 1., 0.]),\n",
       "  tensor([8914, 2467,  705,  ..., 6597, 1743, 4536]),\n",
       "  4636),\n",
       " 'model.decoder.layers.15.fc2.weight': (tensor([1., 0., 0.,  ..., 0., 1., 1.]),\n",
       "  tensor([2060, 1082, 8753,  ..., 8560, 7406, 4837]),\n",
       "  3053),\n",
       " 'model.decoder.layers.14.fc2.weight': (tensor([1., 0., 1.,  ..., 0., 0., 1.]),\n",
       "  tensor([2671, 5783,  942,  ..., 1140, 1364, 4383]),\n",
       "  2371),\n",
       " 'model.decoder.layers.13.fc2.weight': (tensor([0., 0., 0.,  ..., 0., 0., 0.]),\n",
       "  tensor([1479, 1186, 5395,  ..., 7141,  986, 3760]),\n",
       "  1592),\n",
       " 'model.decoder.layers.12.fc2.weight': (tensor([0., 0., 0.,  ..., 0., 0., 0.]),\n",
       "  tensor([2555, 9803, 9028,  ..., 6163, 9427, 4882]),\n",
       "  1522)}"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "layer_masks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "id": "fc0d48da-8f45-4f78-a502-ed4e413b6799",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Line up weights to prune and weights in the state dict\n",
    "mask_index = 0\n",
    "sorted_weight_index = 1\n",
    "pruned_model_params = model_params.copy()\n",
    "\n",
    "for layer_name in layer_masks.keys():\n",
    "    # Prune when nodes are the input\n",
    "    num_nodes_to_drop = int(sum(layer_masks[layer_name][mask_index]).item())\n",
    "    keep_index = torch.sort(layer_masks[layer_name][sorted_weight_index][num_nodes_to_drop:]).values\n",
    "    pruned_input_weights = torch.index_select(pruned_model_params[layer_name], -1, keep_index)\n",
    "    pruned_model_params[layer_name] = pruned_input_weights\n",
    "\n",
    "    # Go to previous layer and prune when nodes are the output\n",
    "    prev_layer_index = params_to_index[layer_name] - 1\n",
    "    prev_layer_name = index_to_params[prev_layer_index]\n",
    "\n",
    "    pruned_output_weights = torch.index_select(pruned_model_params[prev_layer_name], 0, keep_index)\n",
    "    pruned_model_params[prev_layer_name] = pruned_output_weights\n",
    "\n",
    "    # Also do bias term\n",
    "    bias_layer_name = prev_layer_name.replace(\"weight\", \"bias\")\n",
    "    pruned_bias_weights = torch.index_select(pruned_model_params[bias_layer_name], 0, keep_index)\n",
    "    pruned_model_params[bias_layer_name] = pruned_bias_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "id": "fdb4a783-6cd1-44f9-b4e9-a3dd2f5b0156",
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(pruned_model_params, \"pruned_3B_blender_bot2.pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "id": "9b70a7e7-b902-4ecd-8c82-bb0764f3b9fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "state_dict_shapes = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "id": "1e2875a6-35a3-480e-817e-d6cfa7d1a307",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "final_logits_bias torch.Size([1, 8008])\n",
      "model.shared.weight torch.Size([8008, 2560])\n",
      "model.encoder.embed_tokens.weight torch.Size([8008, 2560])\n",
      "model.encoder.embed_positions.weight torch.Size([128, 2560])\n",
      "model.encoder.layers.0.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.encoder.layers.0.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.encoder.layers.0.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.encoder.layers.0.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.encoder.layers.0.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.encoder.layers.0.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.encoder.layers.0.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.encoder.layers.0.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.encoder.layers.0.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.encoder.layers.0.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.encoder.layers.0.fc1.weight torch.Size([24, 2560])\n",
      "model.encoder.layers.0.fc1.bias torch.Size([24])\n",
      "model.encoder.layers.0.fc2.weight torch.Size([2560, 24])\n",
      "model.encoder.layers.0.fc2.bias torch.Size([2560])\n",
      "model.encoder.layers.0.final_layer_norm.weight torch.Size([2560])\n",
      "model.encoder.layers.0.final_layer_norm.bias torch.Size([2560])\n",
      "model.encoder.layers.1.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.encoder.layers.1.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.encoder.layers.1.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.encoder.layers.1.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.encoder.layers.1.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.encoder.layers.1.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.encoder.layers.1.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.encoder.layers.1.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.encoder.layers.1.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.encoder.layers.1.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.encoder.layers.1.fc1.weight torch.Size([90, 2560])\n",
      "model.encoder.layers.1.fc1.bias torch.Size([90])\n",
      "model.encoder.layers.1.fc2.weight torch.Size([2560, 90])\n",
      "model.encoder.layers.1.fc2.bias torch.Size([2560])\n",
      "model.encoder.layers.1.final_layer_norm.weight torch.Size([2560])\n",
      "model.encoder.layers.1.final_layer_norm.bias torch.Size([2560])\n",
      "model.encoder.layer_norm.weight torch.Size([2560])\n",
      "model.encoder.layer_norm.bias torch.Size([2560])\n",
      "model.decoder.embed_tokens.weight torch.Size([8008, 2560])\n",
      "model.decoder.embed_positions.weight torch.Size([128, 2560])\n",
      "model.decoder.layers.0.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.0.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.0.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.0.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.0.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.0.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.0.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.0.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.0.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.0.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.0.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.0.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.0.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.0.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.0.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.0.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.0.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.0.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.0.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.0.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.0.fc1.weight torch.Size([10240, 2560])\n",
      "model.decoder.layers.0.fc1.bias torch.Size([10240])\n",
      "model.decoder.layers.0.fc2.weight torch.Size([2560, 10240])\n",
      "model.decoder.layers.0.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.0.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.0.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.1.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.1.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.1.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.1.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.1.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.1.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.1.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.1.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.1.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.1.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.1.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.1.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.1.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.1.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.1.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.1.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.1.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.1.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.1.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.1.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.1.fc1.weight torch.Size([10240, 2560])\n",
      "model.decoder.layers.1.fc1.bias torch.Size([10240])\n",
      "model.decoder.layers.1.fc2.weight torch.Size([2560, 10240])\n",
      "model.decoder.layers.1.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.1.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.1.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.2.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.2.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.2.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.2.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.2.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.2.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.2.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.2.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.2.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.2.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.2.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.2.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.2.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.2.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.2.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.2.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.2.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.2.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.2.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.2.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.2.fc1.weight torch.Size([10240, 2560])\n",
      "model.decoder.layers.2.fc1.bias torch.Size([10240])\n",
      "model.decoder.layers.2.fc2.weight torch.Size([2560, 10240])\n",
      "model.decoder.layers.2.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.2.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.2.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.3.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.3.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.3.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.3.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.3.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.3.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.3.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.3.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.3.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.3.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.3.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.3.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.3.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.3.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.3.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.3.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.3.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.3.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.3.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.3.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.3.fc1.weight torch.Size([10240, 2560])\n",
      "model.decoder.layers.3.fc1.bias torch.Size([10240])\n",
      "model.decoder.layers.3.fc2.weight torch.Size([2560, 10240])\n",
      "model.decoder.layers.3.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.3.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.3.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.4.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.4.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.4.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.4.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.4.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.4.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.4.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.4.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.4.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.4.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.4.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.4.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.4.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.4.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.4.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.4.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.4.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.4.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.4.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.4.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.4.fc1.weight torch.Size([10240, 2560])\n",
      "model.decoder.layers.4.fc1.bias torch.Size([10240])\n",
      "model.decoder.layers.4.fc2.weight torch.Size([2560, 10240])\n",
      "model.decoder.layers.4.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.4.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.4.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.5.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.5.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.5.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.5.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.5.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.5.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.5.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.5.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.5.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.5.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.5.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.5.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.5.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.5.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.5.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.5.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.5.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.5.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.5.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.5.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.5.fc1.weight torch.Size([10240, 2560])\n",
      "model.decoder.layers.5.fc1.bias torch.Size([10240])\n",
      "model.decoder.layers.5.fc2.weight torch.Size([2560, 10240])\n",
      "model.decoder.layers.5.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.5.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.5.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.6.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.6.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.6.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.6.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.6.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.6.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.6.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.6.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.6.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.6.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.6.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.6.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.6.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.6.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.6.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.6.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.6.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.6.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.6.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.6.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.6.fc1.weight torch.Size([10240, 2560])\n",
      "model.decoder.layers.6.fc1.bias torch.Size([10240])\n",
      "model.decoder.layers.6.fc2.weight torch.Size([2560, 10240])\n",
      "model.decoder.layers.6.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.6.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.6.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.7.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.7.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.7.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.7.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.7.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.7.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.7.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.7.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.7.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.7.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.7.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.7.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.7.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.7.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.7.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.7.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.7.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.7.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.7.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.7.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.7.fc1.weight torch.Size([10240, 2560])\n",
      "model.decoder.layers.7.fc1.bias torch.Size([10240])\n",
      "model.decoder.layers.7.fc2.weight torch.Size([2560, 10240])\n",
      "model.decoder.layers.7.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.7.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.7.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.8.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.8.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.8.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.8.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.8.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.8.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.8.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.8.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.8.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.8.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.8.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.8.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.8.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.8.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.8.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.8.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.8.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.8.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.8.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.8.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.8.fc1.weight torch.Size([10240, 2560])\n",
      "model.decoder.layers.8.fc1.bias torch.Size([10240])\n",
      "model.decoder.layers.8.fc2.weight torch.Size([2560, 10240])\n",
      "model.decoder.layers.8.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.8.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.8.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.9.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.9.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.9.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.9.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.9.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.9.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.9.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.9.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.9.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.9.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.9.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.9.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.9.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.9.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.9.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.9.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.9.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.9.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.9.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.9.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.9.fc1.weight torch.Size([10240, 2560])\n",
      "model.decoder.layers.9.fc1.bias torch.Size([10240])\n",
      "model.decoder.layers.9.fc2.weight torch.Size([2560, 10240])\n",
      "model.decoder.layers.9.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.9.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.9.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.10.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.10.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.10.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.10.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.10.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.10.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.10.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.10.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.10.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.10.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.10.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.10.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.10.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.10.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.10.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.10.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.10.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.10.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.10.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.10.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.10.fc1.weight torch.Size([10240, 2560])\n",
      "model.decoder.layers.10.fc1.bias torch.Size([10240])\n",
      "model.decoder.layers.10.fc2.weight torch.Size([2560, 10240])\n",
      "model.decoder.layers.10.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.10.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.10.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.11.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.11.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.11.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.11.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.11.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.11.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.11.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.11.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.11.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.11.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.11.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.11.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.11.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.11.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.11.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.11.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.11.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.11.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.11.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.11.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.11.fc1.weight torch.Size([10240, 2560])\n",
      "model.decoder.layers.11.fc1.bias torch.Size([10240])\n",
      "model.decoder.layers.11.fc2.weight torch.Size([2560, 10240])\n",
      "model.decoder.layers.11.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.11.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.11.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.12.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.12.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.12.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.12.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.12.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.12.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.12.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.12.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.12.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.12.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.12.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.12.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.12.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.12.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.12.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.12.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.12.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.12.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.12.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.12.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.12.fc1.weight torch.Size([8718, 2560])\n",
      "model.decoder.layers.12.fc1.bias torch.Size([8718])\n",
      "model.decoder.layers.12.fc2.weight torch.Size([2560, 8718])\n",
      "model.decoder.layers.12.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.12.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.12.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.13.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.13.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.13.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.13.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.13.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.13.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.13.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.13.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.13.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.13.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.13.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.13.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.13.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.13.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.13.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.13.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.13.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.13.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.13.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.13.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.13.fc1.weight torch.Size([8648, 2560])\n",
      "model.decoder.layers.13.fc1.bias torch.Size([8648])\n",
      "model.decoder.layers.13.fc2.weight torch.Size([2560, 8648])\n",
      "model.decoder.layers.13.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.13.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.13.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.14.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.14.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.14.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.14.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.14.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.14.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.14.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.14.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.14.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.14.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.14.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.14.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.14.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.14.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.14.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.14.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.14.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.14.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.14.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.14.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.14.fc1.weight torch.Size([7869, 2560])\n",
      "model.decoder.layers.14.fc1.bias torch.Size([7869])\n",
      "model.decoder.layers.14.fc2.weight torch.Size([2560, 7869])\n",
      "model.decoder.layers.14.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.14.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.14.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.15.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.15.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.15.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.15.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.15.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.15.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.15.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.15.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.15.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.15.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.15.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.15.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.15.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.15.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.15.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.15.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.15.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.15.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.15.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.15.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.15.fc1.weight torch.Size([7187, 2560])\n",
      "model.decoder.layers.15.fc1.bias torch.Size([7187])\n",
      "model.decoder.layers.15.fc2.weight torch.Size([2560, 7187])\n",
      "model.decoder.layers.15.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.15.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.15.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.16.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.16.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.16.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.16.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.16.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.16.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.16.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.16.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.16.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.16.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.16.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.16.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.16.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.16.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.16.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.16.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.16.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.16.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.16.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.16.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.16.fc1.weight torch.Size([5604, 2560])\n",
      "model.decoder.layers.16.fc1.bias torch.Size([5604])\n",
      "model.decoder.layers.16.fc2.weight torch.Size([2560, 5604])\n",
      "model.decoder.layers.16.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.16.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.16.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.17.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.17.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.17.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.17.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.17.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.17.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.17.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.17.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.17.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.17.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.17.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.17.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.17.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.17.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.17.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.17.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.17.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.17.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.17.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.17.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.17.fc1.weight torch.Size([4756, 2560])\n",
      "model.decoder.layers.17.fc1.bias torch.Size([4756])\n",
      "model.decoder.layers.17.fc2.weight torch.Size([2560, 4756])\n",
      "model.decoder.layers.17.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.17.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.17.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.18.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.18.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.18.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.18.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.18.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.18.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.18.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.18.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.18.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.18.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.18.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.18.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.18.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.18.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.18.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.18.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.18.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.18.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.18.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.18.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.18.fc1.weight torch.Size([4451, 2560])\n",
      "model.decoder.layers.18.fc1.bias torch.Size([4451])\n",
      "model.decoder.layers.18.fc2.weight torch.Size([2560, 4451])\n",
      "model.decoder.layers.18.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.18.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.18.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.19.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.19.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.19.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.19.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.19.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.19.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.19.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.19.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.19.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.19.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.19.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.19.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.19.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.19.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.19.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.19.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.19.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.19.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.19.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.19.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.19.fc1.weight torch.Size([3992, 2560])\n",
      "model.decoder.layers.19.fc1.bias torch.Size([3992])\n",
      "model.decoder.layers.19.fc2.weight torch.Size([2560, 3992])\n",
      "model.decoder.layers.19.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.19.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.19.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.20.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.20.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.20.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.20.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.20.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.20.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.20.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.20.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.20.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.20.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.20.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.20.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.20.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.20.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.20.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.20.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.20.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.20.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.20.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.20.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.20.fc1.weight torch.Size([3346, 2560])\n",
      "model.decoder.layers.20.fc1.bias torch.Size([3346])\n",
      "model.decoder.layers.20.fc2.weight torch.Size([2560, 3346])\n",
      "model.decoder.layers.20.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.20.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.20.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.21.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.21.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.21.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.21.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.21.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.21.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.21.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.21.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.21.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.21.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.21.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.21.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.21.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.21.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.21.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.21.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.21.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.21.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.21.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.21.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.21.fc1.weight torch.Size([3136, 2560])\n",
      "model.decoder.layers.21.fc1.bias torch.Size([3136])\n",
      "model.decoder.layers.21.fc2.weight torch.Size([2560, 3136])\n",
      "model.decoder.layers.21.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.21.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.21.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.22.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.22.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.22.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.22.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.22.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.22.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.22.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.22.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.22.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.22.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.22.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.22.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.22.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.22.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.22.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.22.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.22.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.22.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.22.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.22.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.22.fc1.weight torch.Size([2667, 2560])\n",
      "model.decoder.layers.22.fc1.bias torch.Size([2667])\n",
      "model.decoder.layers.22.fc2.weight torch.Size([2560, 2667])\n",
      "model.decoder.layers.22.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.22.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.22.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.23.self_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.23.self_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.23.self_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.23.self_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.23.self_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.23.self_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.23.self_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.23.self_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.23.self_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.23.self_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.23.encoder_attn.k_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.23.encoder_attn.k_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.23.encoder_attn.v_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.23.encoder_attn.v_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.23.encoder_attn.q_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.23.encoder_attn.q_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.23.encoder_attn.out_proj.weight torch.Size([2560, 2560])\n",
      "model.decoder.layers.23.encoder_attn.out_proj.bias torch.Size([2560])\n",
      "model.decoder.layers.23.encoder_attn_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.23.encoder_attn_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layers.23.fc1.weight torch.Size([3878, 2560])\n",
      "model.decoder.layers.23.fc1.bias torch.Size([3878])\n",
      "model.decoder.layers.23.fc2.weight torch.Size([2560, 3878])\n",
      "model.decoder.layers.23.fc2.bias torch.Size([2560])\n",
      "model.decoder.layers.23.final_layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layers.23.final_layer_norm.bias torch.Size([2560])\n",
      "model.decoder.layer_norm.weight torch.Size([2560])\n",
      "model.decoder.layer_norm.bias torch.Size([2560])\n",
      "lm_head.weight torch.Size([8008, 2560])\n"
     ]
    }
   ],
   "source": [
    "for param_name in pruned_model_params.keys():\n",
    "    state_dict_shapes[param_name] = pruned_model_params[param_name].shape\n",
    "    print(param_name, pruned_model_params[param_name].shape)\n",
    "    \n",
    "pkl.dump(state_dict_shapes, open(\"pruned_3B_blender_bot2_state_dict_shapes.pkl\", \"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "id": "28af6ba2-5c54-4693-a1ba-56cc2d8da64b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.29669846754807694"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "node_num / len(all_nodes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "id": "81c1d532-4970-4891-88a0-b1110c867364",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "78993"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "node_num"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7207a091-f70d-47c7-9b78-1bc42a328e41",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "beff6b26-2dd6-4776-8cd7-d7551442b756",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tap",
   "language": "python",
   "name": "tap"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}