pruning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "75ec25fe-bed3-43a9-94e8-22413302a7a0",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/homebrew/lib/python3.10/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from node_attribution.node_attribution import get_attributions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "289d2a47-0d93-43ad-897b-3ae05dfcce49",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_size = \"560m\"\n",
    "data = [\"Hello, I am an AlexPrize chatbot\"]\n",
    "prune_percent = 0.30\n",
    "num_params_to_prune = 816115712 * prune_percent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "baf45e79-30ba-4500-8cba-31e5b931a41c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "244834713.6"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_params_to_prune"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8efbe0a3-048c-4ca6-a77e-dfe9315161ad",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished loading bigscience/bloom-560m.\n",
      "                    Num Transformer Blocks: 24\n",
      "                    Num Attention Heads: 16\n",
      "                    Head Dim: 64\n",
      "                    Hidden Size: 1024\n",
      "                    Base Model Param Count: 559,214,592\n",
      "                    Total Param Count (w/ LM Head): 816,115,712\n"
     ]
    }
   ],
   "source": [
    "# Get attributions\n",
    "avg_contributions, max_contributions, model, model_params = get_attributions(model_size, data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "830cf301-0660-43f4-839c-02016f3bb33f",
   "metadata": {},
   "outputs": [],
   "source": [
    "layer_shape_map = {}\n",
    "layer_names, contribution_tensors = zip(*avg_contributions.items())\n",
    "num_layers = len(layer_names)\n",
    "\n",
    "for i in range(num_layers):\n",
    "    layer_name = layer_names[i]\n",
    "    layer_size = contribution_tensors[i].shape[0]\n",
    "    \n",
    "    if i != 0:\n",
    "        next_layer_size = contribution_tensors[i - 1].shape[0]\n",
    "    else:\n",
    "        next_layer_size = 250880\n",
    "        \n",
    "    \n",
    "    if i != (num_layers - 1):\n",
    "        prev_layer_size = contribution_tensors[i + 1].shape[0]\n",
    "    else:\n",
    "        prev_layer_size = 1024\n",
    "        \n",
    "    layer_shape_map[layer_name] = {\n",
    "        \"prev_layer_size\": prev_layer_size,\n",
    "        \"next_layer_size\": next_layer_size,\n",
    "        \"current_layer_size\": layer_size\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "5452d85e-412d-421f-9bfa-530d5b92a097",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'lm_head.weight': {'prev_layer_size': 4096,\n",
       "  'next_layer_size': 250880,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.23.mlp.dense_4h_to_h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 4096},\n",
       " 'transformer.h.23.mlp.dense_h_to_4h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 4096,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.23.self_attention.dense.weight': {'prev_layer_size': 3072,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.23.self_attention.query_key_value_fused_output.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 3072},\n",
       " 'transformer.h.23.self_attention.query_key_value.weight': {'prev_layer_size': 4096,\n",
       "  'next_layer_size': 3072,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.22.mlp.dense_4h_to_h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 4096},\n",
       " 'transformer.h.22.mlp.dense_h_to_4h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 4096,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.22.self_attention.dense.weight': {'prev_layer_size': 3072,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.22.self_attention.query_key_value_fused_output.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 3072},\n",
       " 'transformer.h.22.self_attention.query_key_value.weight': {'prev_layer_size': 4096,\n",
       "  'next_layer_size': 3072,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.21.mlp.dense_4h_to_h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 4096},\n",
       " 'transformer.h.21.mlp.dense_h_to_4h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 4096,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.21.self_attention.dense.weight': {'prev_layer_size': 3072,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.21.self_attention.query_key_value_fused_output.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 3072},\n",
       " 'transformer.h.21.self_attention.query_key_value.weight': {'prev_layer_size': 4096,\n",
       "  'next_layer_size': 3072,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.20.mlp.dense_4h_to_h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 4096},\n",
       " 'transformer.h.20.mlp.dense_h_to_4h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 4096,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.20.self_attention.dense.weight': {'prev_layer_size': 3072,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.20.self_attention.query_key_value_fused_output.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 3072},\n",
       " 'transformer.h.20.self_attention.query_key_value.weight': {'prev_layer_size': 4096,\n",
       "  'next_layer_size': 3072,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.19.mlp.dense_4h_to_h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 4096},\n",
       " 'transformer.h.19.mlp.dense_h_to_4h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 4096,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.19.self_attention.dense.weight': {'prev_layer_size': 3072,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.19.self_attention.query_key_value_fused_output.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 3072},\n",
       " 'transformer.h.19.self_attention.query_key_value.weight': {'prev_layer_size': 4096,\n",
       "  'next_layer_size': 3072,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.18.mlp.dense_4h_to_h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 4096},\n",
       " 'transformer.h.18.mlp.dense_h_to_4h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 4096,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.18.self_attention.dense.weight': {'prev_layer_size': 3072,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.18.self_attention.query_key_value_fused_output.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 3072},\n",
       " 'transformer.h.18.self_attention.query_key_value.weight': {'prev_layer_size': 4096,\n",
       "  'next_layer_size': 3072,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.17.mlp.dense_4h_to_h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 4096},\n",
       " 'transformer.h.17.mlp.dense_h_to_4h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 4096,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.17.self_attention.dense.weight': {'prev_layer_size': 3072,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.17.self_attention.query_key_value_fused_output.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 3072},\n",
       " 'transformer.h.17.self_attention.query_key_value.weight': {'prev_layer_size': 4096,\n",
       "  'next_layer_size': 3072,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.16.mlp.dense_4h_to_h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 4096},\n",
       " 'transformer.h.16.mlp.dense_h_to_4h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 4096,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.16.self_attention.dense.weight': {'prev_layer_size': 3072,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.16.self_attention.query_key_value_fused_output.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 3072},\n",
       " 'transformer.h.16.self_attention.query_key_value.weight': {'prev_layer_size': 4096,\n",
       "  'next_layer_size': 3072,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.15.mlp.dense_4h_to_h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 4096},\n",
       " 'transformer.h.15.mlp.dense_h_to_4h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 4096,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.15.self_attention.dense.weight': {'prev_layer_size': 3072,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.15.self_attention.query_key_value_fused_output.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 3072},\n",
       " 'transformer.h.15.self_attention.query_key_value.weight': {'prev_layer_size': 4096,\n",
       "  'next_layer_size': 3072,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.14.mlp.dense_4h_to_h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 4096},\n",
       " 'transformer.h.14.mlp.dense_h_to_4h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 4096,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.14.self_attention.dense.weight': {'prev_layer_size': 3072,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.14.self_attention.query_key_value_fused_output.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 3072},\n",
       " 'transformer.h.14.self_attention.query_key_value.weight': {'prev_layer_size': 4096,\n",
       "  'next_layer_size': 3072,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.13.mlp.dense_4h_to_h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 4096},\n",
       " 'transformer.h.13.mlp.dense_h_to_4h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 4096,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.13.self_attention.dense.weight': {'prev_layer_size': 3072,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.13.self_attention.query_key_value_fused_output.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 3072},\n",
       " 'transformer.h.13.self_attention.query_key_value.weight': {'prev_layer_size': 4096,\n",
       "  'next_layer_size': 3072,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.12.mlp.dense_4h_to_h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 4096},\n",
       " 'transformer.h.12.mlp.dense_h_to_4h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 4096,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.12.self_attention.dense.weight': {'prev_layer_size': 3072,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.12.self_attention.query_key_value_fused_output.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 3072},\n",
       " 'transformer.h.12.self_attention.query_key_value.weight': {'prev_layer_size': 4096,\n",
       "  'next_layer_size': 3072,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.11.mlp.dense_4h_to_h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 4096},\n",
       " 'transformer.h.11.mlp.dense_h_to_4h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 4096,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.11.self_attention.dense.weight': {'prev_layer_size': 3072,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.11.self_attention.query_key_value_fused_output.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 3072},\n",
       " 'transformer.h.11.self_attention.query_key_value.weight': {'prev_layer_size': 4096,\n",
       "  'next_layer_size': 3072,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.10.mlp.dense_4h_to_h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 4096},\n",
       " 'transformer.h.10.mlp.dense_h_to_4h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 4096,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.10.self_attention.dense.weight': {'prev_layer_size': 3072,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.10.self_attention.query_key_value_fused_output.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 3072},\n",
       " 'transformer.h.10.self_attention.query_key_value.weight': {'prev_layer_size': 4096,\n",
       "  'next_layer_size': 3072,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.9.mlp.dense_4h_to_h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 4096},\n",
       " 'transformer.h.9.mlp.dense_h_to_4h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 4096,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.9.self_attention.dense.weight': {'prev_layer_size': 3072,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.9.self_attention.query_key_value_fused_output.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 3072},\n",
       " 'transformer.h.9.self_attention.query_key_value.weight': {'prev_layer_size': 4096,\n",
       "  'next_layer_size': 3072,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.8.mlp.dense_4h_to_h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 4096},\n",
       " 'transformer.h.8.mlp.dense_h_to_4h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 4096,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.8.self_attention.dense.weight': {'prev_layer_size': 3072,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.8.self_attention.query_key_value_fused_output.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 3072},\n",
       " 'transformer.h.8.self_attention.query_key_value.weight': {'prev_layer_size': 4096,\n",
       "  'next_layer_size': 3072,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.7.mlp.dense_4h_to_h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 4096},\n",
       " 'transformer.h.7.mlp.dense_h_to_4h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 4096,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.7.self_attention.dense.weight': {'prev_layer_size': 3072,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.7.self_attention.query_key_value_fused_output.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 3072},\n",
       " 'transformer.h.7.self_attention.query_key_value.weight': {'prev_layer_size': 4096,\n",
       "  'next_layer_size': 3072,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.6.mlp.dense_4h_to_h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 4096},\n",
       " 'transformer.h.6.mlp.dense_h_to_4h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 4096,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.6.self_attention.dense.weight': {'prev_layer_size': 3072,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.6.self_attention.query_key_value_fused_output.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 3072},\n",
       " 'transformer.h.6.self_attention.query_key_value.weight': {'prev_layer_size': 4096,\n",
       "  'next_layer_size': 3072,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.5.mlp.dense_4h_to_h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 4096},\n",
       " 'transformer.h.5.mlp.dense_h_to_4h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 4096,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.5.self_attention.dense.weight': {'prev_layer_size': 3072,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.5.self_attention.query_key_value_fused_output.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 3072},\n",
       " 'transformer.h.5.self_attention.query_key_value.weight': {'prev_layer_size': 4096,\n",
       "  'next_layer_size': 3072,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.4.mlp.dense_4h_to_h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 4096},\n",
       " 'transformer.h.4.mlp.dense_h_to_4h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 4096,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.4.self_attention.dense.weight': {'prev_layer_size': 3072,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.4.self_attention.query_key_value_fused_output.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 3072},\n",
       " 'transformer.h.4.self_attention.query_key_value.weight': {'prev_layer_size': 4096,\n",
       "  'next_layer_size': 3072,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.3.mlp.dense_4h_to_h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 4096},\n",
       " 'transformer.h.3.mlp.dense_h_to_4h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 4096,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.3.self_attention.dense.weight': {'prev_layer_size': 3072,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.3.self_attention.query_key_value_fused_output.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 3072},\n",
       " 'transformer.h.3.self_attention.query_key_value.weight': {'prev_layer_size': 4096,\n",
       "  'next_layer_size': 3072,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.2.mlp.dense_4h_to_h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 4096},\n",
       " 'transformer.h.2.mlp.dense_h_to_4h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 4096,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.2.self_attention.dense.weight': {'prev_layer_size': 3072,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.2.self_attention.query_key_value_fused_output.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 3072},\n",
       " 'transformer.h.2.self_attention.query_key_value.weight': {'prev_layer_size': 4096,\n",
       "  'next_layer_size': 3072,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.1.mlp.dense_4h_to_h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 4096},\n",
       " 'transformer.h.1.mlp.dense_h_to_4h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 4096,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.1.self_attention.dense.weight': {'prev_layer_size': 3072,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.1.self_attention.query_key_value_fused_output.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 3072},\n",
       " 'transformer.h.1.self_attention.query_key_value.weight': {'prev_layer_size': 4096,\n",
       "  'next_layer_size': 3072,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.0.mlp.dense_4h_to_h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 4096},\n",
       " 'transformer.h.0.mlp.dense_h_to_4h.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 4096,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.0.self_attention.dense.weight': {'prev_layer_size': 3072,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 1024},\n",
       " 'transformer.h.0.self_attention.query_key_value_fused_output.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 1024,\n",
       "  'current_layer_size': 3072},\n",
       " 'transformer.h.0.self_attention.query_key_value.weight': {'prev_layer_size': 1024,\n",
       "  'next_layer_size': 3072,\n",
       "  'current_layer_size': 1024}}"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "layer_shape_map\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "52304b74-b166-4c13-a95f-2eafc0c3a705",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "all_nodes = []\n",
    "\n",
    "for layer_name, contribution_tensor in avg_contributions.items():\n",
    "    for node_id, node in enumerate(contribution_tensor.tolist()):\n",
    "        node_name = f\"{layer_name}.{node_id}\"\n",
    "        all_nodes.append((node_name, node))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "fee0d3a6-0b98-4e18-9940-34b0f3ed7db4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sort all the nodes\n",
    "all_nodes.sort(key = lambda x:x[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "2e88c002-42ec-43f3-b3e3-0a027ccd99f6",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub data rate exceeded.\n",
      "The Jupyter server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--ServerApp.iopub_data_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "ServerApp.iopub_data_rate_limit=1000000.0 (bytes/sec)\n",
      "ServerApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Iterate through nodes to make the prune list. \n",
    "# Whenever a node is added to the prune list, check total number or params being \n",
    "nodes_to_prune = []\n",
    "num_params_pruned = 0\n",
    "node_num = 0\n",
    "\n",
    "while num_params_pruned < num_params_to_prune:\n",
    "    node = all_nodes[node_num]\n",
    "    node_name = node[0]\n",
    "    layer_name = \".\".join(node_name.split(\".\")[:-1])\n",
    "    print(layer_name)\n",
    "                     \n",
    "    nodes_to_prune.append(node)\n",
    "    shapes = layer_shape_map[layer_name]\n",
    "    num_params_pruned += shapes[\"prev_layer_size\"]\n",
    "    num_params_pruned += shapes[\"next_layer_size\"]\n",
    "    \n",
    "    node_num += 1\n",
    "    \n",
    "print(node_num)\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "28af6ba2-5c54-4693-a1ba-56cc2d8da64b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.22894920254149378"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "node_num / len(all_nodes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "66bcda99-683d-4f35-825c-28b71907d0c6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "56501"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "node_num"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "d1883898-3b6c-4460-83e1-a63892d0809c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('transformer.h.22.self_attention.query_key_value_fused_output.weight.1787',\n",
       "  -17626.154296875),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1851',\n",
       "  -17626.154296875),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1807',\n",
       "  -5429.0966796875),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1743',\n",
       "  -5429.09619140625),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2898',\n",
       "  -4951.94775390625),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2962',\n",
       "  -4951.947265625),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1785',\n",
       "  -4597.2646484375),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1849',\n",
       "  -4597.2646484375),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1779',\n",
       "  -4437.36474609375),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1843',\n",
       "  -4437.36474609375),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1823',\n",
       "  -3873.46533203125),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1759',\n",
       "  -3873.46484375),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1765',\n",
       "  -3773.039794921875),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1829',\n",
       "  -3773.039794921875),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2985',\n",
       "  -3555.57763671875),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2921',\n",
       "  -3555.577392578125),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1767',\n",
       "  -3279.81787109375),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1831',\n",
       "  -3279.817626953125),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1772',\n",
       "  -2869.59619140625),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1836',\n",
       "  -2869.595947265625),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1790',\n",
       "  -2600.72802734375),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1854',\n",
       "  -2600.7275390625),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2960',\n",
       "  -2419.749267578125),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2896',\n",
       "  -2419.748779296875),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2881',\n",
       "  -2298.41943359375),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2945',\n",
       "  -2298.41943359375),\n",
       " ('transformer.h.23.self_attention.query_key_value_fused_output.weight.2143',\n",
       "  -2280.5712890625),\n",
       " ('transformer.h.23.self_attention.query_key_value_fused_output.weight.2207',\n",
       "  -2280.5712890625),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1731',\n",
       "  -1638.811279296875),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1795',\n",
       "  -1638.8111572265625),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1750',\n",
       "  -1631.107177734375),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1814',\n",
       "  -1631.107177734375),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2892',\n",
       "  -1608.1104736328125),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2956',\n",
       "  -1608.1103515625),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2737',\n",
       "  -1531.256591796875),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2801',\n",
       "  -1531.256591796875),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2528',\n",
       "  -1359.215576171875),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2592',\n",
       "  -1359.215576171875),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2515',\n",
       "  -1338.11572265625),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2579',\n",
       "  -1338.1156005859375),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2367',\n",
       "  -1285.348876953125),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2431',\n",
       "  -1285.3487548828125),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2995',\n",
       "  -1284.8709716796875),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2931',\n",
       "  -1284.870849609375),\n",
       " ('transformer.h.15.self_attention.query_key_value_fused_output.weight.2749',\n",
       "  -1281.872802734375),\n",
       " ('transformer.h.15.self_attention.query_key_value_fused_output.weight.2813',\n",
       "  -1281.8726806640625),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2739',\n",
       "  -1142.407470703125),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2803',\n",
       "  -1142.4073486328125),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2511',\n",
       "  -1134.9927978515625),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2575',\n",
       "  -1134.99267578125),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2356',\n",
       "  -1134.658935546875),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2420',\n",
       "  -1134.658935546875),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1783',\n",
       "  -1121.8349609375),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1847',\n",
       "  -1121.8348388671875),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2566',\n",
       "  -1118.895751953125),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2502',\n",
       "  -1118.8956298828125),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.1741',\n",
       "  -1101.218017578125),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.1805',\n",
       "  -1101.218017578125),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2711',\n",
       "  -1087.4814453125),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2775',\n",
       "  -1087.4814453125),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2751',\n",
       "  -1069.98193359375),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2815',\n",
       "  -1069.98193359375),\n",
       " ('transformer.h.15.self_attention.query_key_value_fused_output.weight.1935',\n",
       "  -1063.133544921875),\n",
       " ('transformer.h.15.self_attention.query_key_value_fused_output.weight.1999',\n",
       "  -1063.1334228515625),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2604',\n",
       "  -1060.0281982421875),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2540',\n",
       "  -1060.028076171875),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2521',\n",
       "  -1053.316650390625),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2585',\n",
       "  -1053.316650390625),\n",
       " ('transformer.h.15.self_attention.query_key_value_fused_output.weight.2741',\n",
       "  -1006.5369262695312),\n",
       " ('transformer.h.15.self_attention.query_key_value_fused_output.weight.2805',\n",
       "  -1006.5369262695312),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1736',\n",
       "  -984.3743286132812),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1800',\n",
       "  -984.3743286132812),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2882',\n",
       "  -949.1866455078125),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2946',\n",
       "  -949.1865234375),\n",
       " ('transformer.h.15.self_attention.query_key_value_fused_output.weight.2017',\n",
       "  -845.7488403320312),\n",
       " ('transformer.h.15.self_attention.query_key_value_fused_output.weight.1953',\n",
       "  -845.7487182617188),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1842',\n",
       "  -824.6135864257812),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1778',\n",
       "  -824.6134643554688),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.1925',\n",
       "  -803.2000732421875),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.1989',\n",
       "  -803.2000122070312),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.1977',\n",
       "  -768.5006713867188),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.2041',\n",
       "  -768.5006713867188),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.1781',\n",
       "  -763.9976196289062),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.1845',\n",
       "  -763.99755859375),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2167',\n",
       "  -762.0789184570312),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2231',\n",
       "  -762.0789184570312),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2313',\n",
       "  -741.4217529296875),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2377',\n",
       "  -741.4217529296875),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.1980',\n",
       "  -736.7418212890625),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.2044',\n",
       "  -736.7417602539062),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1932',\n",
       "  -702.626953125),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1996',\n",
       "  -702.6268920898438),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.3002',\n",
       "  -700.4070434570312),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2938',\n",
       "  -700.4069213867188),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2516',\n",
       "  -674.4959716796875),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2580',\n",
       "  -674.4959716796875),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1788',\n",
       "  -672.3343505859375),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1852',\n",
       "  -672.3342895507812),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1752',\n",
       "  -646.5599975585938),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1816',\n",
       "  -646.5599975585938),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2751',\n",
       "  -630.7921752929688),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2815',\n",
       "  -630.7921142578125),\n",
       " ('transformer.h.15.self_attention.query_key_value_fused_output.weight.2040',\n",
       "  -623.3056030273438),\n",
       " ('transformer.h.15.self_attention.query_key_value_fused_output.weight.1976',\n",
       "  -623.3055419921875),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2163',\n",
       "  -607.1151123046875),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2227',\n",
       "  -607.1151123046875),\n",
       " ('transformer.h.17.self_attention.query_key_value_fused_output.weight.1763',\n",
       "  -605.91259765625),\n",
       " ('transformer.h.17.self_attention.query_key_value_fused_output.weight.1827',\n",
       "  -605.9125366210938),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.1827',\n",
       "  -563.0584716796875),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.1763',\n",
       "  -563.0584106445312),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2156',\n",
       "  -559.2550048828125),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2220',\n",
       "  -559.2550048828125),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2519',\n",
       "  -536.4290161132812),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2583',\n",
       "  -536.4288940429688),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1991',\n",
       "  -534.6528930664062),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1927',\n",
       "  -534.65283203125),\n",
       " ('transformer.h.12.self_attention.query_key_value_fused_output.weight.2974',\n",
       "  -519.5216064453125),\n",
       " ('transformer.h.12.self_attention.query_key_value_fused_output.weight.2910',\n",
       "  -519.5215454101562),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.1775',\n",
       "  -514.7926025390625),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.1839',\n",
       "  -514.7926025390625),\n",
       " ('transformer.h.17.self_attention.query_key_value_fused_output.weight.2519',\n",
       "  -503.972900390625),\n",
       " ('transformer.h.17.self_attention.query_key_value_fused_output.weight.2583',\n",
       "  -503.972900390625),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2517',\n",
       "  -500.2974548339844),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2581',\n",
       "  -500.2974548339844),\n",
       " ('transformer.h.15.self_attention.query_key_value_fused_output.weight.1946',\n",
       "  -492.51300048828125),\n",
       " ('transformer.h.15.self_attention.query_key_value_fused_output.weight.2010',\n",
       "  -492.51300048828125),\n",
       " ('transformer.h.17.self_attention.query_key_value_fused_output.weight.1768',\n",
       "  -491.5577087402344),\n",
       " ('transformer.h.17.self_attention.query_key_value_fused_output.weight.1832',\n",
       "  -491.5577087402344),\n",
       " ('transformer.h.23.self_attention.query_key_value_fused_output.weight.2174',\n",
       "  -483.13531494140625),\n",
       " ('transformer.h.23.self_attention.query_key_value_fused_output.weight.2238',\n",
       "  -483.13531494140625),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.2228',\n",
       "  -477.2222595214844),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.2164',\n",
       "  -477.22222900390625),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.1984',\n",
       "  -463.63983154296875),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.1920',\n",
       "  -463.6398010253906),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.1789',\n",
       "  -455.2906188964844),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.1853',\n",
       "  -455.2906188964844),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2720',\n",
       "  -451.47235107421875),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2784',\n",
       "  -451.4723205566406),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2695',\n",
       "  -449.40460205078125),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2759',\n",
       "  -449.40460205078125),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1751',\n",
       "  -430.36151123046875),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1815',\n",
       "  -430.36151123046875),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1804',\n",
       "  -421.231201171875),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1740',\n",
       "  -421.23114013671875),\n",
       " ('transformer.h.17.self_attention.query_key_value_fused_output.weight.1788',\n",
       "  -421.10711669921875),\n",
       " ('transformer.h.17.self_attention.query_key_value_fused_output.weight.1852',\n",
       "  -421.10711669921875),\n",
       " ('transformer.h.17.self_attention.query_key_value_fused_output.weight.2560',\n",
       "  -415.0045471191406),\n",
       " ('transformer.h.17.self_attention.query_key_value_fused_output.weight.2496',\n",
       "  -415.0044860839844),\n",
       " ('transformer.h.16.self_attention.query_key_value_fused_output.weight.1813',\n",
       "  -410.5751037597656),\n",
       " ('transformer.h.16.self_attention.query_key_value_fused_output.weight.1749',\n",
       "  -410.5750427246094),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.1982',\n",
       "  -398.394775390625),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2046',\n",
       "  -398.394775390625),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.2154',\n",
       "  -397.43218994140625),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.2218',\n",
       "  -397.43218994140625),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.3001',\n",
       "  -396.44775390625),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.2937',\n",
       "  -396.4477233886719),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1930',\n",
       "  -393.71337890625),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1994',\n",
       "  -393.7133483886719),\n",
       " ('transformer.h.15.self_attention.query_key_value_fused_output.weight.2910',\n",
       "  -390.4688415527344),\n",
       " ('transformer.h.15.self_attention.query_key_value_fused_output.weight.2974',\n",
       "  -390.4688415527344),\n",
       " ('transformer.h.16.self_attention.query_key_value_fused_output.weight.2316',\n",
       "  -379.74005126953125),\n",
       " ('transformer.h.16.self_attention.query_key_value_fused_output.weight.2380',\n",
       "  -379.74005126953125),\n",
       " ('transformer.h.16.self_attention.query_key_value_fused_output.weight.1758',\n",
       "  -373.6001892089844),\n",
       " ('transformer.h.16.self_attention.query_key_value_fused_output.weight.1822',\n",
       "  -373.60015869140625),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2039',\n",
       "  -373.1600646972656),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1975',\n",
       "  -373.1600036621094),\n",
       " ('transformer.h.17.self_attention.query_key_value_fused_output.weight.1843',\n",
       "  -372.5393371582031),\n",
       " ('transformer.h.17.self_attention.query_key_value_fused_output.weight.1779',\n",
       "  -372.539306640625),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.1598',\n",
       "  -364.44122314453125),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.1662',\n",
       "  -364.44122314453125),\n",
       " ('transformer.h.13.self_attention.query_key_value_fused_output.weight.1731',\n",
       "  -360.9054260253906),\n",
       " ('transformer.h.13.self_attention.query_key_value_fused_output.weight.1795',\n",
       "  -360.9054260253906),\n",
       " ('transformer.h.13.self_attention.query_key_value_fused_output.weight.2738',\n",
       "  -359.0048522949219),\n",
       " ('transformer.h.13.self_attention.query_key_value_fused_output.weight.2802',\n",
       "  -359.00482177734375),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2146',\n",
       "  -357.56207275390625),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2210',\n",
       "  -357.56207275390625),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.1382',\n",
       "  -353.736572265625),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.1446',\n",
       "  -353.7365417480469),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.1814',\n",
       "  -352.7231140136719),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.1750',\n",
       "  -352.72308349609375),\n",
       " ('transformer.h.13.self_attention.query_key_value_fused_output.weight.2127',\n",
       "  -337.77142333984375),\n",
       " ('transformer.h.13.self_attention.query_key_value_fused_output.weight.2191',\n",
       "  -337.77142333984375),\n",
       " ('transformer.h.15.self_attention.query_key_value_fused_output.weight.2409',\n",
       "  -334.4547119140625),\n",
       " ('transformer.h.15.self_attention.query_key_value_fused_output.weight.2345',\n",
       "  -334.45465087890625),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2690',\n",
       "  -332.003662109375),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2754',\n",
       "  -332.0036315917969),\n",
       " ('transformer.h.13.self_attention.query_key_value_fused_output.weight.2763',\n",
       "  -331.9072570800781),\n",
       " ('transformer.h.13.self_attention.query_key_value_fused_output.weight.2699',\n",
       "  -331.9072265625),\n",
       " ('transformer.h.20.mlp.dense_h_to_4h.weight.12', -331.56573486328125),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.1832',\n",
       "  -330.94842529296875),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.1768',\n",
       "  -330.9483947753906),\n",
       " ('transformer.h.3.self_attention.query_key_value_fused_output.weight.2323',\n",
       "  -329.94451904296875),\n",
       " ('transformer.h.3.self_attention.query_key_value_fused_output.weight.2387',\n",
       "  -329.94451904296875),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2731',\n",
       "  -316.79046630859375),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2795',\n",
       "  -316.79046630859375),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1844',\n",
       "  -315.9661865234375),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1780',\n",
       "  -315.9661560058594),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.1762',\n",
       "  -315.95770263671875),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.1826',\n",
       "  -315.9576721191406),\n",
       " ('transformer.h.15.self_attention.query_key_value_fused_output.weight.2746',\n",
       "  -313.9896545410156),\n",
       " ('transformer.h.15.self_attention.query_key_value_fused_output.weight.2810',\n",
       "  -313.9896545410156),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1771',\n",
       "  -310.7879638671875),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1835',\n",
       "  -310.78790283203125),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2748',\n",
       "  -309.70806884765625),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2812',\n",
       "  -309.70806884765625),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2689',\n",
       "  -309.6002197265625),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2753',\n",
       "  -309.6001892089844),\n",
       " ('transformer.h.13.self_attention.query_key_value_fused_output.weight.1777',\n",
       "  -309.0290222167969),\n",
       " ('transformer.h.13.self_attention.query_key_value_fused_output.weight.1841',\n",
       "  -309.0290222167969),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.1561',\n",
       "  -306.7158203125),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.1625',\n",
       "  -306.7158203125),\n",
       " ('transformer.h.13.self_attention.query_key_value_fused_output.weight.2169',\n",
       "  -305.8760070800781),\n",
       " ('transformer.h.13.self_attention.query_key_value_fused_output.weight.2233',\n",
       "  -305.8760070800781),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2383',\n",
       "  -298.34283447265625),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2319',\n",
       "  -298.3428039550781),\n",
       " ('transformer.h.12.self_attention.query_key_value_fused_output.weight.2162',\n",
       "  -297.33148193359375),\n",
       " ('transformer.h.12.self_attention.query_key_value_fused_output.weight.2226',\n",
       "  -297.3314514160156),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2698',\n",
       "  -296.8390808105469),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2762',\n",
       "  -296.83905029296875),\n",
       " ('transformer.h.13.mlp.dense_h_to_4h.weight.12', -291.44024658203125),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.1920',\n",
       "  -289.6966247558594),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.1984',\n",
       "  -289.69659423828125),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2694',\n",
       "  -289.2681884765625),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2758',\n",
       "  -289.2681579589844),\n",
       " ('transformer.h.17.self_attention.query_key_value_fused_output.weight.1838',\n",
       "  -283.3248291015625),\n",
       " ('transformer.h.17.self_attention.query_key_value_fused_output.weight.1774',\n",
       "  -283.3247985839844),\n",
       " ('transformer.h.3.self_attention.query_key_value_fused_output.weight.2318',\n",
       "  -283.1988830566406),\n",
       " ('transformer.h.3.self_attention.query_key_value_fused_output.weight.2382',\n",
       "  -283.1988525390625),\n",
       " ('transformer.h.14.mlp.dense_h_to_4h.weight.12', -281.4267578125),\n",
       " ('transformer.h.16.self_attention.query_key_value_fused_output.weight.2751',\n",
       "  -281.2855529785156),\n",
       " ('transformer.h.16.self_attention.query_key_value_fused_output.weight.2815',\n",
       "  -281.2855224609375),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1748',\n",
       "  -279.41192626953125),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1812',\n",
       "  -279.41192626953125),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.1577',\n",
       "  -275.5144958496094),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.1641',\n",
       "  -275.51446533203125),\n",
       " ('transformer.h.12.self_attention.query_key_value_fused_output.weight.2902',\n",
       "  -273.67645263671875),\n",
       " ('transformer.h.12.self_attention.query_key_value_fused_output.weight.2966',\n",
       "  -273.67645263671875),\n",
       " ('transformer.h.21.mlp.dense_h_to_4h.weight.12', -271.68719482421875),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.1378',\n",
       "  -267.7469787597656),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.1442',\n",
       "  -267.7469482421875),\n",
       " ('transformer.h.20.mlp.dense_h_to_4h.weight.269', -258.68426513671875),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2523',\n",
       "  -257.8341064453125),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2587',\n",
       "  -257.8341064453125),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2139',\n",
       "  -254.38363647460938),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2203',\n",
       "  -254.38363647460938),\n",
       " ('transformer.h.17.self_attention.query_key_value_fused_output.weight.1818',\n",
       "  -253.22085571289062),\n",
       " ('transformer.h.17.self_attention.query_key_value_fused_output.weight.1754',\n",
       "  -253.2208251953125),\n",
       " ('transformer.h.15.mlp.dense_h_to_4h.weight.12', -251.873779296875),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.2131',\n",
       "  -251.12747192382812),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.2195',\n",
       "  -251.12747192382812),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.1747',\n",
       "  -248.2505340576172),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.1811',\n",
       "  -248.25051879882812),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.1156',\n",
       "  -246.74673461914062),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.1220',\n",
       "  -246.74673461914062),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.1348',\n",
       "  -244.93429565429688),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.1412',\n",
       "  -244.93429565429688),\n",
       " ('transformer.h.16.self_attention.query_key_value_fused_output.weight.1738',\n",
       "  -243.272705078125),\n",
       " ('transformer.h.16.self_attention.query_key_value_fused_output.weight.1802',\n",
       "  -243.27267456054688),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2501',\n",
       "  -243.0608367919922),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2565',\n",
       "  -243.06082153320312),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2349',\n",
       "  -242.1268768310547),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2413',\n",
       "  -242.1268768310547),\n",
       " ('transformer.h.15.self_attention.query_key_value_fused_output.weight.2898',\n",
       "  -240.70364379882812),\n",
       " ('transformer.h.15.self_attention.query_key_value_fused_output.weight.2962',\n",
       "  -240.70361328125),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2513',\n",
       "  -238.9315643310547),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2577',\n",
       "  -238.9315643310547),\n",
       " ('transformer.h.8.self_attention.query_key_value_fused_output.weight.2897',\n",
       "  -238.4648895263672),\n",
       " ('transformer.h.8.self_attention.query_key_value_fused_output.weight.2961',\n",
       "  -238.46487426757812),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2509',\n",
       "  -235.43612670898438),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2573',\n",
       "  -235.43612670898438),\n",
       " ('transformer.h.18.mlp.dense_h_to_4h.weight.12', -235.13510131835938),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2013',\n",
       "  -232.4154052734375),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.1949',\n",
       "  -232.41537475585938),\n",
       " ('transformer.h.18.mlp.dense_h_to_4h.weight.269', -232.1708221435547),\n",
       " ('transformer.h.17.mlp.dense_h_to_4h.weight.12', -231.91757202148438),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.1752',\n",
       "  -231.8057403564453),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.1816',\n",
       "  -231.8057403564453),\n",
       " ('transformer.h.14.self_attention.query_key_value_fused_output.weight.2803',\n",
       "  -230.1488494873047),\n",
       " ('transformer.h.14.self_attention.query_key_value_fused_output.weight.2739',\n",
       "  -230.14883422851562),\n",
       " ('transformer.h.13.self_attention.query_key_value_fused_output.weight.2132',\n",
       "  -227.5021209716797),\n",
       " ('transformer.h.13.self_attention.query_key_value_fused_output.weight.2196',\n",
       "  -227.5021209716797),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2801',\n",
       "  -223.1957244873047),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2737',\n",
       "  -223.19570922851562),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2811',\n",
       "  -222.0131072998047),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2747',\n",
       "  -222.01309204101562),\n",
       " ('transformer.h.19.mlp.dense_h_to_4h.weight.12', -220.3076171875),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2720',\n",
       "  -217.2096405029297),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2784',\n",
       "  -217.2096405029297),\n",
       " ('transformer.h.15.self_attention.query_key_value_fused_output.weight.2935',\n",
       "  -216.5596923828125),\n",
       " ('transformer.h.15.self_attention.query_key_value_fused_output.weight.2999',\n",
       "  -216.5596923828125),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2740',\n",
       "  -215.7218017578125),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2804',\n",
       "  -215.7218017578125),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2587',\n",
       "  -214.4370880126953),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2523',\n",
       "  -214.4370574951172),\n",
       " ('transformer.h.14.self_attention.query_key_value_fused_output.weight.2723',\n",
       "  -212.4458770751953),\n",
       " ('transformer.h.14.self_attention.query_key_value_fused_output.weight.2787',\n",
       "  -212.4458465576172),\n",
       " ('transformer.h.16.self_attention.query_key_value_fused_output.weight.2331',\n",
       "  -210.5235595703125),\n",
       " ('transformer.h.16.self_attention.query_key_value_fused_output.weight.2395',\n",
       "  -210.5235595703125),\n",
       " ('transformer.h.12.mlp.dense_h_to_4h.weight.12', -210.48104858398438),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1813',\n",
       "  -210.32315063476562),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1749',\n",
       "  -210.3231201171875),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1757',\n",
       "  -209.0543212890625),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1821',\n",
       "  -209.0543212890625),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2528',\n",
       "  -208.9544677734375),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2592',\n",
       "  -208.9544677734375),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.2915',\n",
       "  -207.84872436523438),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.2979',\n",
       "  -207.84872436523438),\n",
       " ('transformer.h.13.mlp.dense_h_to_4h.weight.269', -206.201416015625),\n",
       " ('transformer.h.21.mlp.dense_h_to_4h.weight.269', -195.5046844482422),\n",
       " ('transformer.h.12.self_attention.query_key_value_fused_output.weight.1212',\n",
       "  -195.38394165039062),\n",
       " ('transformer.h.12.self_attention.query_key_value_fused_output.weight.1276',\n",
       "  -195.38392639160156),\n",
       " ('transformer.h.17.mlp.dense_h_to_4h.weight.269', -194.86001586914062),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2795',\n",
       "  -191.46458435058594),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2731',\n",
       "  -191.46456909179688),\n",
       " ('transformer.h.16.self_attention.query_key_value_fused_output.weight.1842',\n",
       "  -190.71507263183594),\n",
       " ('transformer.h.16.self_attention.query_key_value_fused_output.weight.1778',\n",
       "  -190.71505737304688),\n",
       " ('transformer.h.21.mlp.dense_h_to_4h.weight.785', -189.1331024169922),\n",
       " ('transformer.h.12.self_attention.query_key_value_fused_output.weight.2152',\n",
       "  -187.25601196289062),\n",
       " ('transformer.h.12.self_attention.query_key_value_fused_output.weight.2216',\n",
       "  -187.25599670410156),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2618',\n",
       "  -187.1393585205078),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2554',\n",
       "  -187.13934326171875),\n",
       " ('transformer.h.6.self_attention.query_key_value_fused_output.weight.2768',\n",
       "  -185.669677734375),\n",
       " ('transformer.h.6.self_attention.query_key_value_fused_output.weight.2704',\n",
       "  -185.66966247558594),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2792',\n",
       "  -185.52015686035156),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2728',\n",
       "  -185.52012634277344),\n",
       " ('transformer.h.17.self_attention.query_key_value_fused_output.weight.2890',\n",
       "  -180.24932861328125),\n",
       " ('transformer.h.17.self_attention.query_key_value_fused_output.weight.2954',\n",
       "  -180.24932861328125),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2924',\n",
       "  -178.85101318359375),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2988',\n",
       "  -178.85101318359375),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2139',\n",
       "  -178.35733032226562),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2203',\n",
       "  -178.35733032226562),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2891',\n",
       "  -178.2902374267578),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2955',\n",
       "  -178.2902374267578),\n",
       " ('transformer.h.14.mlp.dense_h_to_4h.weight.269', -177.7452850341797),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.2503',\n",
       "  -171.98854064941406),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.2567',\n",
       "  -171.98854064941406),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1960',\n",
       "  -171.67184448242188),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2024',\n",
       "  -171.67184448242188),\n",
       " ('transformer.h.19.mlp.dense_h_to_4h.weight.269', -171.0489959716797),\n",
       " ('transformer.h.15.mlp.dense_h_to_4h.weight.269', -169.56333923339844),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2522',\n",
       "  -169.2610626220703),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2586',\n",
       "  -169.2610626220703),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2744',\n",
       "  -168.5738067626953),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2808',\n",
       "  -168.5738067626953),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2310',\n",
       "  -168.54446411132812),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2374',\n",
       "  -168.54444885253906),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2305',\n",
       "  -167.48965454101562),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2369',\n",
       "  -167.48965454101562),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2946',\n",
       "  -165.35353088378906),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2882',\n",
       "  -165.353515625),\n",
       " ('transformer.h.16.self_attention.query_key_value_fused_output.weight.2364',\n",
       "  -162.09359741210938),\n",
       " ('transformer.h.16.self_attention.query_key_value_fused_output.weight.2428',\n",
       "  -162.0935821533203),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2777',\n",
       "  -160.62779235839844),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2713',\n",
       "  -160.62777709960938),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2813',\n",
       "  -159.58782958984375),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2749',\n",
       "  -159.58779907226562),\n",
       " ('transformer.h.8.self_attention.query_key_value_fused_output.weight.2930',\n",
       "  -159.40682983398438),\n",
       " ('transformer.h.8.self_attention.query_key_value_fused_output.weight.2994',\n",
       "  -159.4068145751953),\n",
       " ('transformer.h.8.self_attention.query_key_value_fused_output.weight.2957',\n",
       "  -158.05821228027344),\n",
       " ('transformer.h.8.self_attention.query_key_value_fused_output.weight.2893',\n",
       "  -158.05819702148438),\n",
       " ('transformer.h.17.self_attention.query_key_value_fused_output.weight.2917',\n",
       "  -156.80264282226562),\n",
       " ('transformer.h.17.self_attention.query_key_value_fused_output.weight.2981',\n",
       "  -156.80264282226562),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2905',\n",
       "  -154.89527893066406),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2969',\n",
       "  -154.89527893066406),\n",
       " ('transformer.h.16.mlp.dense_h_to_4h.weight.12', -153.56346130371094),\n",
       " ('transformer.h.13.self_attention.query_key_value_fused_output.weight.2719',\n",
       "  -153.29031372070312),\n",
       " ('transformer.h.13.self_attention.query_key_value_fused_output.weight.2783',\n",
       "  -153.29031372070312),\n",
       " ('transformer.h.8.self_attention.query_key_value_fused_output.weight.2884',\n",
       "  -153.16424560546875),\n",
       " ('transformer.h.8.self_attention.query_key_value_fused_output.weight.2948',\n",
       "  -153.16424560546875),\n",
       " ('transformer.h.12.mlp.dense_h_to_4h.weight.269', -148.48336791992188),\n",
       " ('transformer.h.14.mlp.dense_h_to_4h.weight.35', -147.31626892089844),\n",
       " ('transformer.h.20.mlp.dense_h_to_4h.weight.785', -147.19981384277344),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.1645',\n",
       "  -146.08279418945312),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.1581',\n",
       "  -146.08277893066406),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2525',\n",
       "  -142.88302612304688),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2589',\n",
       "  -142.88302612304688),\n",
       " ('transformer.h.17.self_attention.query_key_value_fused_output.weight.1740',\n",
       "  -141.42892456054688),\n",
       " ('transformer.h.17.self_attention.query_key_value_fused_output.weight.1804',\n",
       "  -141.42892456054688),\n",
       " ('transformer.h.21.mlp.dense_h_to_4h.weight.391', -141.0735321044922),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.1556',\n",
       "  -140.96092224121094),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.1620',\n",
       "  -140.96092224121094),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.1541',\n",
       "  -140.6151885986328),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.1605',\n",
       "  -140.61517333984375),\n",
       " ('transformer.h.16.self_attention.query_key_value_fused_output.weight.1964',\n",
       "  -139.29135131835938),\n",
       " ('transformer.h.16.self_attention.query_key_value_fused_output.weight.2028',\n",
       "  -139.29132080078125),\n",
       " ('transformer.h.0.mlp.dense_h_to_4h.weight.996', -139.22549438476562),\n",
       " ('transformer.h.22.mlp.dense_h_to_4h.weight.12', -138.03182983398438),\n",
       " ('transformer.h.15.self_attention.query_key_value_fused_output.weight.1927',\n",
       "  -137.23287963867188),\n",
       " ('transformer.h.15.self_attention.query_key_value_fused_output.weight.1991',\n",
       "  -137.23287963867188),\n",
       " ('transformer.h.23.self_attention.query_key_value_fused_output.weight.2148',\n",
       "  -136.15139770507812),\n",
       " ('transformer.h.23.self_attention.query_key_value_fused_output.weight.2212',\n",
       "  -136.15139770507812),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.1364',\n",
       "  -135.58663940429688),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.1428',\n",
       "  -135.58663940429688),\n",
       " ('transformer.h.8.self_attention.query_key_value_fused_output.weight.2910',\n",
       "  -133.3311309814453),\n",
       " ('transformer.h.8.self_attention.query_key_value_fused_output.weight.2974',\n",
       "  -133.3311309814453),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2118',\n",
       "  -132.26931762695312),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2182',\n",
       "  -132.26931762695312),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2925',\n",
       "  -131.98219299316406),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2989',\n",
       "  -131.98219299316406),\n",
       " ('transformer.h.6.self_attention.query_key_value_fused_output.weight.2727',\n",
       "  -131.7188262939453),\n",
       " ('transformer.h.6.self_attention.query_key_value_fused_output.weight.2791',\n",
       "  -131.7188262939453),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2731',\n",
       "  -130.58937072753906),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2795',\n",
       "  -130.58935546875),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2725',\n",
       "  -130.55638122558594),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2789',\n",
       "  -130.55638122558594),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2496',\n",
       "  -130.50372314453125),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2560',\n",
       "  -130.50372314453125),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2392',\n",
       "  -127.85542297363281),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2328',\n",
       "  -127.85540771484375),\n",
       " ('transformer.h.17.mlp.dense_h_to_4h.weight.35', -127.7209243774414),\n",
       " ('transformer.h.6.self_attention.query_key_value_fused_output.weight.2697',\n",
       "  -125.29693603515625),\n",
       " ('transformer.h.6.self_attention.query_key_value_fused_output.weight.2761',\n",
       "  -125.29693603515625),\n",
       " ('transformer.h.13.mlp.dense_h_to_4h.weight.35', -124.91374206542969),\n",
       " ('transformer.h.10.mlp.dense_h_to_4h.weight.12', -124.8724365234375),\n",
       " ('transformer.h.10.self_attention.query_key_value_fused_output.weight.774',\n",
       "  -124.7378921508789),\n",
       " ('transformer.h.10.self_attention.query_key_value_fused_output.weight.838',\n",
       "  -124.7378921508789),\n",
       " ('transformer.h.17.self_attention.query_key_value_fused_output.weight.2582',\n",
       "  -124.63033294677734),\n",
       " ('transformer.h.17.self_attention.query_key_value_fused_output.weight.2518',\n",
       "  -124.63032531738281),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2917',\n",
       "  -124.39634704589844),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2981',\n",
       "  -124.39634704589844),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2734',\n",
       "  -121.99117279052734),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2798',\n",
       "  -121.99117279052734),\n",
       " ('transformer.h.20.mlp.dense_h_to_4h.weight.391', -119.45899963378906),\n",
       " ('transformer.h.16.self_attention.query_key_value_fused_output.weight.2701',\n",
       "  -118.91340637207031),\n",
       " ('transformer.h.16.self_attention.query_key_value_fused_output.weight.2765',\n",
       "  -118.91337585449219),\n",
       " ('transformer.h.7.self_attention.query_key_value_fused_output.weight.2802',\n",
       "  -118.59001159667969),\n",
       " ('transformer.h.7.self_attention.query_key_value_fused_output.weight.2738',\n",
       "  -118.59000396728516),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.1790',\n",
       "  -117.8899154663086),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.1854',\n",
       "  -117.889892578125),\n",
       " ('transformer.h.23.self_attention.query_key_value_fused_output.weight.1377',\n",
       "  -117.23500061035156),\n",
       " ('transformer.h.23.self_attention.query_key_value_fused_output.weight.1441',\n",
       "  -117.2349853515625),\n",
       " ('transformer.h.12.self_attention.query_key_value_fused_output.weight.2337',\n",
       "  -116.95991516113281),\n",
       " ('transformer.h.12.self_attention.query_key_value_fused_output.weight.2401',\n",
       "  -116.95991516113281),\n",
       " ('transformer.h.15.mlp.dense_h_to_4h.weight.35', -115.855712890625),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2360',\n",
       "  -114.35909271240234),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2424',\n",
       "  -114.35908508300781),\n",
       " ('transformer.h.6.self_attention.query_key_value_fused_output.weight.2920',\n",
       "  -113.89013671875),\n",
       " ('transformer.h.6.self_attention.query_key_value_fused_output.weight.2984',\n",
       "  -113.89013671875),\n",
       " ('transformer.h.22.mlp.dense_h_to_4h.weight.391', -113.142578125),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2537',\n",
       "  -112.26605224609375),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2601',\n",
       "  -112.26605224609375),\n",
       " ('transformer.h.8.self_attention.query_key_value_fused_output.weight.2938',\n",
       "  -112.08465576171875),\n",
       " ('transformer.h.8.self_attention.query_key_value_fused_output.weight.3002',\n",
       "  -112.08465576171875),\n",
       " ('transformer.h.17.self_attention.query_key_value_fused_output.weight.1798',\n",
       "  -109.85420989990234),\n",
       " ('transformer.h.17.self_attention.query_key_value_fused_output.weight.1734',\n",
       "  -109.85420227050781),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2740',\n",
       "  -109.53619384765625),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2804',\n",
       "  -109.53617858886719),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1830',\n",
       "  -109.44136810302734),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1766',\n",
       "  -109.44136047363281),\n",
       " ('transformer.h.23.self_attention.query_key_value_fused_output.weight.2614',\n",
       "  -108.53948974609375),\n",
       " ('transformer.h.23.self_attention.query_key_value_fused_output.weight.2550',\n",
       "  -108.53947448730469),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1755',\n",
       "  -107.27641296386719),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1819',\n",
       "  -107.27641296386719),\n",
       " ('transformer.h.16.mlp.dense_h_to_4h.weight.269', -107.2747802734375),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2520',\n",
       "  -107.05926513671875),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2584',\n",
       "  -107.05924987792969),\n",
       " ('transformer.h.15.self_attention.query_key_value_fused_output.weight.1982',\n",
       "  -106.8737564086914),\n",
       " ('transformer.h.15.self_attention.query_key_value_fused_output.weight.2046',\n",
       "  -106.8737564086914),\n",
       " ('transformer.h.10.mlp.dense_h_to_4h.weight.269', -106.40788269042969),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.1938',\n",
       "  -106.27518463134766),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2002',\n",
       "  -106.27518463134766),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.1976',\n",
       "  -105.45149230957031),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2040',\n",
       "  -105.45149230957031),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.2168',\n",
       "  -104.7005615234375),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.2232',\n",
       "  -104.70054626464844),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2733',\n",
       "  -103.85194396972656),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2797',\n",
       "  -103.85194396972656),\n",
       " ('transformer.h.17.self_attention.query_key_value_fused_output.weight.1185',\n",
       "  -103.73307037353516),\n",
       " ('transformer.h.17.self_attention.query_key_value_fused_output.weight.1249',\n",
       "  -103.7330551147461),\n",
       " ('transformer.h.18.mlp.dense_h_to_4h.weight.785', -103.0750732421875),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1768',\n",
       "  -102.51448822021484),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1832',\n",
       "  -102.51448059082031),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2135',\n",
       "  -102.5063705444336),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2199',\n",
       "  -102.5063705444336),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2751',\n",
       "  -102.1321792602539),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2815',\n",
       "  -102.13217163085938),\n",
       " ('transformer.h.0.self_attention.query_key_value_fused_output.weight.1829',\n",
       "  -101.72748565673828),\n",
       " ('transformer.h.0.self_attention.query_key_value_fused_output.weight.1765',\n",
       "  -101.72747802734375),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2551',\n",
       "  -101.43894958496094),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2615',\n",
       "  -101.4389419555664),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2548',\n",
       "  -101.31937408447266),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2612',\n",
       "  -101.3193588256836),\n",
       " ('transformer.h.7.self_attention.query_key_value_fused_output.weight.2743',\n",
       "  -101.11548614501953),\n",
       " ('transformer.h.7.self_attention.query_key_value_fused_output.weight.2807',\n",
       "  -101.115478515625),\n",
       " ('transformer.h.11.mlp.dense_h_to_4h.weight.12', -100.937255859375),\n",
       " ('transformer.h.15.self_attention.query_key_value_fused_output.weight.1554',\n",
       "  -100.80793762207031),\n",
       " ('transformer.h.15.self_attention.query_key_value_fused_output.weight.1618',\n",
       "  -100.80793762207031),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.2556',\n",
       "  -100.0355453491211),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.2620',\n",
       "  -100.0355453491211),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2515',\n",
       "  -99.5872573852539),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2579',\n",
       "  -99.58724975585938),\n",
       " ('transformer.h.23.self_attention.query_key_value_fused_output.weight.2144',\n",
       "  -98.91973114013672),\n",
       " ('transformer.h.23.self_attention.query_key_value_fused_output.weight.2208',\n",
       "  -98.91972351074219),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1786',\n",
       "  -97.67660522460938),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1850',\n",
       "  -97.67659759521484),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2894',\n",
       "  -97.21341705322266),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2958',\n",
       "  -97.21340942382812),\n",
       " ('transformer.h.8.self_attention.query_key_value_fused_output.weight.2534',\n",
       "  -96.29593658447266),\n",
       " ('transformer.h.8.self_attention.query_key_value_fused_output.weight.2598',\n",
       "  -96.29593658447266),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2734',\n",
       "  -95.45626831054688),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2798',\n",
       "  -95.45626068115234),\n",
       " ('transformer.h.6.self_attention.query_key_value_fused_output.weight.2896',\n",
       "  -95.2567367553711),\n",
       " ('transformer.h.6.self_attention.query_key_value_fused_output.weight.2960',\n",
       "  -95.25672912597656),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2116',\n",
       "  -95.18220520019531),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2180',\n",
       "  -95.18220520019531),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2569',\n",
       "  -94.62553405761719),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2505',\n",
       "  -94.62552642822266),\n",
       " ('transformer.h.9.self_attention.query_key_value_fused_output.weight.2362',\n",
       "  -94.41862487792969),\n",
       " ('transformer.h.9.self_attention.query_key_value_fused_output.weight.2426',\n",
       "  -94.41862487792969),\n",
       " ('transformer.h.19.mlp.dense_h_to_4h.weight.785', -90.93270111083984),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.2731',\n",
       "  -90.6970443725586),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.2795',\n",
       "  -90.6970443725586),\n",
       " ('transformer.h.1.mlp.dense_h_to_4h.weight.996', -90.20327758789062),\n",
       " ('transformer.h.5.self_attention.query_key_value_fused_output.weight.2702',\n",
       "  -89.98698425292969),\n",
       " ('transformer.h.5.self_attention.query_key_value_fused_output.weight.2766',\n",
       "  -89.98697662353516),\n",
       " ('transformer.h.5.self_attention.query_key_value_fused_output.weight.2722',\n",
       "  -89.84788513183594),\n",
       " ('transformer.h.5.self_attention.query_key_value_fused_output.weight.2786',\n",
       "  -89.8478775024414),\n",
       " ('transformer.h.5.self_attention.query_key_value_fused_output.weight.2710',\n",
       "  -89.55354309082031),\n",
       " ('transformer.h.5.self_attention.query_key_value_fused_output.weight.2774',\n",
       "  -89.55354309082031),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2529',\n",
       "  -89.2066421508789),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2593',\n",
       "  -89.20662689208984),\n",
       " ('transformer.h.5.self_attention.query_key_value_fused_output.weight.2701',\n",
       "  -89.19772338867188),\n",
       " ('transformer.h.5.self_attention.query_key_value_fused_output.weight.2765',\n",
       "  -89.19771575927734),\n",
       " ('transformer.h.21.mlp.dense_h_to_4h.weight.557', -88.98902893066406),\n",
       " ('transformer.h.8.self_attention.query_key_value_fused_output.weight.2170',\n",
       "  -88.54835510253906),\n",
       " ('transformer.h.8.self_attention.query_key_value_fused_output.weight.2234',\n",
       "  -88.54835510253906),\n",
       " ('transformer.h.5.self_attention.query_key_value_fused_output.weight.2731',\n",
       "  -88.36002349853516),\n",
       " ('transformer.h.5.self_attention.query_key_value_fused_output.weight.2795',\n",
       "  -88.36002349853516),\n",
       " ('transformer.h.21.mlp.dense_h_to_4h.weight.544', -87.90898895263672),\n",
       " ('transformer.h.14.self_attention.query_key_value_fused_output.weight.2746',\n",
       "  -86.72596740722656),\n",
       " ('transformer.h.14.self_attention.query_key_value_fused_output.weight.2810',\n",
       "  -86.7259521484375),\n",
       " ('transformer.h.23.self_attention.query_key_value_fused_output.weight.1979',\n",
       "  -86.70459747314453),\n",
       " ('transformer.h.23.self_attention.query_key_value_fused_output.weight.2043',\n",
       "  -86.70458984375),\n",
       " ('transformer.h.16.self_attention.query_key_value_fused_output.weight.1798',\n",
       "  -86.64878845214844),\n",
       " ('transformer.h.16.self_attention.query_key_value_fused_output.weight.1734',\n",
       "  -86.6487808227539),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2574',\n",
       "  -86.58551025390625),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2510',\n",
       "  -86.58550262451172),\n",
       " ('transformer.h.5.self_attention.query_key_value_fused_output.weight.2700',\n",
       "  -86.48368835449219),\n",
       " ('transformer.h.5.self_attention.query_key_value_fused_output.weight.2764',\n",
       "  -86.48368835449219),\n",
       " ('transformer.h.20.mlp.dense_h_to_4h.weight.544', -86.20355224609375),\n",
       " ('transformer.h.22.mlp.dense_h_to_4h.weight.431', -86.14450073242188),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.1387',\n",
       "  -85.92881774902344),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.1451',\n",
       "  -85.92881774902344),\n",
       " ('transformer.h.13.mlp.dense_h_to_4h.weight.494', -85.83634948730469),\n",
       " ('transformer.h.13.mlp.dense_h_to_4h.weight.996', -85.7934341430664),\n",
       " ('transformer.h.2.mlp.dense_h_to_4h.weight.996', -85.49565124511719),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2504',\n",
       "  -84.94316864013672),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2568',\n",
       "  -84.94316101074219),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2607',\n",
       "  -84.54219818115234),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2543',\n",
       "  -84.54219055175781),\n",
       " ('transformer.h.5.self_attention.query_key_value_fused_output.weight.2712',\n",
       "  -84.0946273803711),\n",
       " ('transformer.h.5.self_attention.query_key_value_fused_output.weight.2776',\n",
       "  -84.09461975097656),\n",
       " ('transformer.h.6.self_attention.query_key_value_fused_output.weight.2777',\n",
       "  -83.77630615234375),\n",
       " ('transformer.h.6.self_attention.query_key_value_fused_output.weight.2713',\n",
       "  -83.77629852294922),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2145',\n",
       "  -83.72179412841797),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2209',\n",
       "  -83.72179412841797),\n",
       " ('transformer.h.6.self_attention.query_key_value_fused_output.weight.2917',\n",
       "  -83.47505187988281),\n",
       " ('transformer.h.6.self_attention.query_key_value_fused_output.weight.2981',\n",
       "  -83.47505187988281),\n",
       " ('transformer.h.4.self_attention.query_key_value_fused_output.weight.2592',\n",
       "  -83.15150451660156),\n",
       " ('transformer.h.4.self_attention.query_key_value_fused_output.weight.2528',\n",
       "  -83.15149688720703),\n",
       " ('transformer.h.6.self_attention.query_key_value_fused_output.weight.2755',\n",
       "  -82.31383514404297),\n",
       " ('transformer.h.6.self_attention.query_key_value_fused_output.weight.2691',\n",
       "  -82.31382751464844),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.1980',\n",
       "  -82.04718017578125),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2044',\n",
       "  -82.04716491699219),\n",
       " ('transformer.h.7.self_attention.query_key_value_fused_output.weight.2727',\n",
       "  -81.61386108398438),\n",
       " ('transformer.h.7.self_attention.query_key_value_fused_output.weight.2791',\n",
       "  -81.61386108398438),\n",
       " ('transformer.h.19.mlp.dense_h_to_4h.weight.35', -81.41238403320312),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2732',\n",
       "  -80.94783020019531),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2796',\n",
       "  -80.94783020019531),\n",
       " ('transformer.h.13.self_attention.query_key_value_fused_output.weight.2227',\n",
       "  -80.71762084960938),\n",
       " ('transformer.h.13.self_attention.query_key_value_fused_output.weight.2163',\n",
       "  -80.71761322021484),\n",
       " ('transformer.h.3.mlp.dense_h_to_4h.weight.996', -80.41809844970703),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1760',\n",
       "  -79.91281127929688),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1824',\n",
       "  -79.91280364990234),\n",
       " ('transformer.h.7.self_attention.query_key_value_fused_output.weight.2767',\n",
       "  -79.5831527709961),\n",
       " ('transformer.h.7.self_attention.query_key_value_fused_output.weight.2703',\n",
       "  -79.58314514160156),\n",
       " ('transformer.h.6.self_attention.query_key_value_fused_output.weight.2812',\n",
       "  -79.21871185302734),\n",
       " ('transformer.h.6.self_attention.query_key_value_fused_output.weight.2748',\n",
       "  -79.21870422363281),\n",
       " ('transformer.h.12.mlp.dense_h_to_4h.weight.785', -78.82610321044922),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2596',\n",
       "  -78.75038146972656),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2532',\n",
       "  -78.75037384033203),\n",
       " ('transformer.h.16.self_attention.query_key_value_fused_output.weight.1772',\n",
       "  -78.67528533935547),\n",
       " ('transformer.h.16.self_attention.query_key_value_fused_output.weight.1836',\n",
       "  -78.67528533935547),\n",
       " ('transformer.h.17.mlp.dense_h_to_4h.weight.785', -78.46696472167969),\n",
       " ('transformer.h.9.self_attention.query_key_value_fused_output.weight.2706',\n",
       "  -77.51811218261719),\n",
       " ('transformer.h.9.self_attention.query_key_value_fused_output.weight.2770',\n",
       "  -77.51811218261719),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2900',\n",
       "  -77.48561096191406),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2964',\n",
       "  -77.48561096191406),\n",
       " ('transformer.h.14.mlp.dense_h_to_4h.weight.996', -76.93913269042969),\n",
       " ('transformer.h.7.mlp.dense_h_to_4h.weight.12', -76.79487609863281),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2172',\n",
       "  -76.74773406982422),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2236',\n",
       "  -76.74772644042969),\n",
       " ('transformer.h.8.self_attention.query_key_value_fused_output.weight.2906',\n",
       "  -76.68098449707031),\n",
       " ('transformer.h.8.self_attention.query_key_value_fused_output.weight.2970',\n",
       "  -76.68098449707031),\n",
       " ('transformer.h.21.mlp.dense_h_to_4h.weight.76', -76.60105895996094),\n",
       " ('transformer.h.23.self_attention.query_key_value_fused_output.weight.1462',\n",
       "  -76.46110534667969),\n",
       " ('transformer.h.23.self_attention.query_key_value_fused_output.weight.1398',\n",
       "  -76.46109771728516),\n",
       " ('transformer.h.6.self_attention.query_key_value_fused_output.weight.1733',\n",
       "  -76.40431213378906),\n",
       " ('transformer.h.6.self_attention.query_key_value_fused_output.weight.1797',\n",
       "  -76.40431213378906),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.2577',\n",
       "  -76.32820129394531),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.2513',\n",
       "  -76.32817840576172),\n",
       " ('transformer.h.6.self_attention.query_key_value_fused_output.weight.2895',\n",
       "  -76.16466522216797),\n",
       " ('transformer.h.6.self_attention.query_key_value_fused_output.weight.2959',\n",
       "  -76.16465759277344),\n",
       " ('transformer.h.3.self_attention.query_key_value_fused_output.weight.1360',\n",
       "  -76.11009216308594),\n",
       " ('transformer.h.3.self_attention.query_key_value_fused_output.weight.1424',\n",
       "  -76.11009216308594),\n",
       " ('transformer.h.13.mlp.dense_h_to_4h.weight.785', -76.10014343261719),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1806',\n",
       "  -75.73162078857422),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1742',\n",
       "  -75.73161315917969),\n",
       " ('transformer.h.11.mlp.dense_h_to_4h.weight.269', -75.64756774902344),\n",
       " ('transformer.h.16.mlp.dense_h_to_4h.weight.785', -75.56529998779297),\n",
       " ('transformer.h.4.self_attention.query_key_value_fused_output.weight.2556',\n",
       "  -75.1051025390625),\n",
       " ('transformer.h.4.self_attention.query_key_value_fused_output.weight.2620',\n",
       "  -75.1051025390625),\n",
       " ('transformer.h.9.self_attention.query_key_value_fused_output.weight.1954',\n",
       "  -74.93297576904297),\n",
       " ('transformer.h.9.self_attention.query_key_value_fused_output.weight.2018',\n",
       "  -74.93297576904297),\n",
       " ('transformer.h.12.mlp.dense_h_to_4h.weight.494', -74.74144744873047),\n",
       " ('transformer.h.0.self_attention.query_key_value_fused_output.weight.1730',\n",
       "  -74.69559478759766),\n",
       " ('transformer.h.0.self_attention.query_key_value_fused_output.weight.1794',\n",
       "  -74.69558715820312),\n",
       " ('transformer.h.9.self_attention.query_key_value_fused_output.weight.2339',\n",
       "  -74.50680541992188),\n",
       " ('transformer.h.9.self_attention.query_key_value_fused_output.weight.2403',\n",
       "  -74.50679779052734),\n",
       " ('transformer.h.22.mlp.dense_h_to_4h.weight.785', -74.45365905761719),\n",
       " ('transformer.h.23.mlp.dense_h_to_4h.weight.164', -74.24531555175781),\n",
       " ('transformer.h.10.self_attention.query_key_value_fused_output.weight.2929',\n",
       "  -73.99873352050781),\n",
       " ('transformer.h.10.self_attention.query_key_value_fused_output.weight.2993',\n",
       "  -73.99873352050781),\n",
       " ('transformer.h.5.mlp.dense_h_to_4h.weight.269', -73.73164367675781),\n",
       " ('transformer.h.6.mlp.dense_h_to_4h.weight.544', -73.53752136230469),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2936',\n",
       "  -73.43266296386719),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.3000',\n",
       "  -73.43266296386719),\n",
       " ('transformer.h.2.self_attention.query_key_value_fused_output.weight.1436',\n",
       "  -73.41756439208984),\n",
       " ('transformer.h.2.self_attention.query_key_value_fused_output.weight.1372',\n",
       "  -73.41755676269531),\n",
       " ('transformer.h.5.self_attention.query_key_value_fused_output.weight.2930',\n",
       "  -73.37492370605469),\n",
       " ('transformer.h.5.self_attention.query_key_value_fused_output.weight.2994',\n",
       "  -73.37492370605469),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2532',\n",
       "  -73.35931396484375),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2596',\n",
       "  -73.35929870605469),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2366',\n",
       "  -73.35102844238281),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2430',\n",
       "  -73.35102844238281),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2342',\n",
       "  -73.31729888916016),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2406',\n",
       "  -73.31729125976562),\n",
       " ('transformer.h.21.mlp.dense_h_to_4h.weight.84', -72.6951675415039),\n",
       " ('transformer.h.4.self_attention.query_key_value_fused_output.weight.2504',\n",
       "  -72.65289306640625),\n",
       " ('transformer.h.4.self_attention.query_key_value_fused_output.weight.2568',\n",
       "  -72.65289306640625),\n",
       " ('transformer.h.10.mlp.dense_h_to_4h.weight.35', -71.8910140991211),\n",
       " ('transformer.h.4.mlp.dense_h_to_4h.weight.269', -71.7379379272461),\n",
       " ('transformer.h.6.self_attention.query_key_value_fused_output.weight.2923',\n",
       "  -71.56982421875),\n",
       " ('transformer.h.6.self_attention.query_key_value_fused_output.weight.2987',\n",
       "  -71.56982421875),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.2117',\n",
       "  -71.27954864501953),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.2181',\n",
       "  -71.279541015625),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2928',\n",
       "  -71.2269287109375),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2992',\n",
       "  -71.2269287109375),\n",
       " ('transformer.h.19.mlp.dense_h_to_4h.weight.996', -71.03327941894531),\n",
       " ('transformer.h.4.self_attention.query_key_value_fused_output.weight.2501',\n",
       "  -71.00118255615234),\n",
       " ('transformer.h.4.self_attention.query_key_value_fused_output.weight.2565',\n",
       "  -71.00118255615234),\n",
       " ('transformer.h.20.mlp.dense_h_to_4h.weight.84', -70.92430114746094),\n",
       " ('transformer.h.15.mlp.dense_h_to_4h.weight.996', -70.89353942871094),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2950',\n",
       "  -70.84089660644531),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2886',\n",
       "  -70.84088897705078),\n",
       " ('transformer.h.23.mlp.dense_h_to_4h.weight.431', -70.79888916015625),\n",
       " ('transformer.h.5.self_attention.query_key_value_fused_output.weight.2733',\n",
       "  -70.73693084716797),\n",
       " ('transformer.h.5.self_attention.query_key_value_fused_output.weight.2797',\n",
       "  -70.73693084716797),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2348',\n",
       "  -70.66387939453125),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2412',\n",
       "  -70.66387176513672),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2545',\n",
       "  -70.292236328125),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2609',\n",
       "  -70.29222869873047),\n",
       " ('transformer.h.20.mlp.dense_h_to_4h.weight.557', -70.17816925048828),\n",
       " ('transformer.h.4.self_attention.query_key_value_fused_output.weight.2537',\n",
       "  -70.1063461303711),\n",
       " ('transformer.h.4.self_attention.query_key_value_fused_output.weight.2601',\n",
       "  -70.1063461303711),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.819',\n",
       "  -70.02790832519531),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.883',\n",
       "  -70.02790832519531),\n",
       " ('transformer.h.15.mlp.dense_h_to_4h.weight.84', -69.81817626953125),\n",
       " ('transformer.h.12.self_attention.query_key_value_fused_output.weight.1199',\n",
       "  -69.7612533569336),\n",
       " ('transformer.h.12.self_attention.query_key_value_fused_output.weight.1263',\n",
       "  -69.7612533569336),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.2368',\n",
       "  -69.61002349853516),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.2304',\n",
       "  -69.61001586914062),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1732',\n",
       "  -69.38922119140625),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1796',\n",
       "  -69.38921356201172),\n",
       " ('transformer.h.6.self_attention.query_key_value_fused_output.weight.1353',\n",
       "  -69.3612060546875),\n",
       " ('transformer.h.6.self_attention.query_key_value_fused_output.weight.1417',\n",
       "  -69.3612060546875),\n",
       " ('transformer.h.14.self_attention.query_key_value_fused_output.weight.2754',\n",
       "  -69.14005279541016),\n",
       " ('transformer.h.14.self_attention.query_key_value_fused_output.weight.2690',\n",
       "  -69.1400375366211),\n",
       " ('transformer.h.23.self_attention.query_key_value_fused_output.weight.2007',\n",
       "  -69.05972290039062),\n",
       " ('transformer.h.23.self_attention.query_key_value_fused_output.weight.1943',\n",
       "  -69.0597152709961),\n",
       " ('transformer.h.6.self_attention.query_key_value_fused_output.weight.2741',\n",
       "  -68.97516632080078),\n",
       " ('transformer.h.6.self_attention.query_key_value_fused_output.weight.2805',\n",
       "  -68.97516632080078),\n",
       " ('transformer.h.11.mlp.dense_h_to_4h.weight.35', -68.96389770507812),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2356',\n",
       "  -68.921630859375),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2420',\n",
       "  -68.92161560058594),\n",
       " ('transformer.h.9.self_attention.query_key_value_fused_output.weight.2787',\n",
       "  -68.70570373535156),\n",
       " ('transformer.h.9.self_attention.query_key_value_fused_output.weight.2723',\n",
       "  -68.70569610595703),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1791',\n",
       "  -68.5947265625),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1855',\n",
       "  -68.59471893310547),\n",
       " ('transformer.h.3.mlp.dense_h_to_4h.weight.269', -68.58982849121094),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2788',\n",
       "  -68.47506713867188),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2724',\n",
       "  -68.47505950927734),\n",
       " ('transformer.h.23.mlp.dense_h_to_4h.weight.391', -68.33917999267578),\n",
       " ('transformer.h.22.mlp.dense_h_to_4h.weight.338', -68.04845428466797),\n",
       " ('transformer.h.14.mlp.dense_h_to_4h.weight.494', -67.94481658935547),\n",
       " ('transformer.h.14.self_attention.query_key_value_fused_output.weight.2966',\n",
       "  -67.69145202636719),\n",
       " ('transformer.h.14.self_attention.query_key_value_fused_output.weight.2902',\n",
       "  -67.69144439697266),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.1742',\n",
       "  -67.59635925292969),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.1806',\n",
       "  -67.59634399414062),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.1958',\n",
       "  -67.56987762451172),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2022',\n",
       "  -67.56986999511719),\n",
       " ('transformer.h.7.mlp.dense_h_to_4h.weight.269', -67.39556121826172),\n",
       " ('transformer.h.18.mlp.dense_h_to_4h.weight.391', -66.85804748535156),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.2311',\n",
       "  -66.80762481689453),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.2375',\n",
       "  -66.80762481689453),\n",
       " ('transformer.h.10.self_attention.query_key_value_fused_output.weight.2942',\n",
       "  -66.63790130615234),\n",
       " ('transformer.h.10.self_attention.query_key_value_fused_output.weight.3006',\n",
       "  -66.63790130615234),\n",
       " ('transformer.h.9.self_attention.query_key_value_fused_output.weight.2315',\n",
       "  -66.41001892089844),\n",
       " ('transformer.h.9.self_attention.query_key_value_fused_output.weight.2379',\n",
       "  -66.4100112915039),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2332',\n",
       "  -66.14396667480469),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2396',\n",
       "  -66.14396667480469),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.1174',\n",
       "  -66.09797668457031),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.1238',\n",
       "  -66.09797668457031),\n",
       " ('transformer.h.13.self_attention.query_key_value_fused_output.weight.876',\n",
       "  -66.0032958984375),\n",
       " ('transformer.h.13.self_attention.query_key_value_fused_output.weight.812',\n",
       "  -66.00328826904297),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2497',\n",
       "  -65.98413848876953),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2561',\n",
       "  -65.984130859375),\n",
       " ('transformer.h.18.mlp.dense_h_to_4h.weight.996', -65.69441223144531),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.1972',\n",
       "  -65.69377136230469),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2036',\n",
       "  -65.69376373291016),\n",
       " ('transformer.h.23.self_attention.query_key_value_fused_output.weight.2175',\n",
       "  -65.62849426269531),\n",
       " ('transformer.h.23.self_attention.query_key_value_fused_output.weight.2239',\n",
       "  -65.62849426269531),\n",
       " ('transformer.h.4.self_attention.query_key_value_fused_output.weight.2512',\n",
       "  -65.26588439941406),\n",
       " ('transformer.h.4.self_attention.query_key_value_fused_output.weight.2576',\n",
       "  -65.26587677001953),\n",
       " ('transformer.h.17.mlp.dense_h_to_4h.weight.996', -65.16475677490234),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2791',\n",
       "  -64.97525024414062),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2727',\n",
       "  -64.9752426147461),\n",
       " ('transformer.h.23.self_attention.query_key_value_fused_output.weight.1854',\n",
       "  -64.8491439819336),\n",
       " ('transformer.h.23.self_attention.query_key_value_fused_output.weight.1790',\n",
       "  -64.84913635253906),\n",
       " ('transformer.h.7.mlp.dense_h_to_4h.weight.544', -64.64646911621094),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2520',\n",
       "  -64.4450454711914),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2584',\n",
       "  -64.44503784179688),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.2536',\n",
       "  -64.42066955566406),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.2600',\n",
       "  -64.42066955566406),\n",
       " ('transformer.h.16.self_attention.query_key_value_fused_output.weight.1054',\n",
       "  -64.24185943603516),\n",
       " ('transformer.h.16.self_attention.query_key_value_fused_output.weight.990',\n",
       "  -64.24185180664062),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.1540',\n",
       "  -63.8393669128418),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.1604',\n",
       "  -63.83936309814453),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2548',\n",
       "  -63.7568359375),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2612',\n",
       "  -63.75682830810547),\n",
       " ('transformer.h.6.mlp.dense_h_to_4h.weight.269', -63.662757873535156),\n",
       " ('transformer.h.13.mlp.dense_h_to_4h.weight.84', -63.606834411621094),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.1960',\n",
       "  -63.27465057373047),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.2024',\n",
       "  -63.27465057373047),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2910',\n",
       "  -62.94952392578125),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2974',\n",
       "  -62.94952392578125),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2555',\n",
       "  -62.891807556152344),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2619',\n",
       "  -62.891807556152344),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2531',\n",
       "  -62.76787567138672),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2595',\n",
       "  -62.76787185668945),\n",
       " ('transformer.h.4.self_attention.query_key_value_fused_output.weight.2605',\n",
       "  -62.75788497924805),\n",
       " ('transformer.h.4.self_attention.query_key_value_fused_output.weight.2541',\n",
       "  -62.75788116455078),\n",
       " ('transformer.h.17.self_attention.query_key_value_fused_output.weight.1797',\n",
       "  -62.3067512512207),\n",
       " ('transformer.h.17.self_attention.query_key_value_fused_output.weight.1733',\n",
       "  -62.30674362182617),\n",
       " ('transformer.h.6.self_attention.query_key_value_fused_output.weight.1753',\n",
       "  -62.28937530517578),\n",
       " ('transformer.h.6.self_attention.query_key_value_fused_output.weight.1817',\n",
       "  -62.28936767578125),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.1562',\n",
       "  -61.969139099121094),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.1626',\n",
       "  -61.969139099121094),\n",
       " ('transformer.h.14.mlp.dense_h_to_4h.weight.84', -61.84650802612305),\n",
       " ('transformer.h.4.mlp.dense_h_to_4h.weight.12', -61.843116760253906),\n",
       " ('transformer.h.12.self_attention.query_key_value_fused_output.weight.2174',\n",
       "  -61.7664909362793),\n",
       " ('transformer.h.12.self_attention.query_key_value_fused_output.weight.2238',\n",
       "  -61.7664909362793),\n",
       " ('transformer.h.12.self_attention.query_key_value_fused_output.weight.2339',\n",
       "  -61.20867919921875),\n",
       " ('transformer.h.12.self_attention.query_key_value_fused_output.weight.2403',\n",
       "  -61.20867919921875),\n",
       " ('transformer.h.23.mlp.dense_h_to_4h.weight.644', -61.00415802001953),\n",
       " ('transformer.h.15.mlp.dense_h_to_4h.weight.785', -60.78764724731445),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2547',\n",
       "  -60.66697311401367),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2611',\n",
       "  -60.666969299316406),\n",
       " ('transformer.h.6.mlp.dense_h_to_4h.weight.996', -60.08442306518555),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2717',\n",
       "  -60.059898376464844),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2781',\n",
       "  -60.059898376464844),\n",
       " ('transformer.h.23.mlp.dense_h_to_4h.weight.927', -59.97407913208008),\n",
       " ('transformer.h.4.self_attention.query_key_value_fused_output.weight.2913',\n",
       "  -59.70389938354492),\n",
       " ('transformer.h.4.self_attention.query_key_value_fused_output.weight.2977',\n",
       "  -59.70389938354492),\n",
       " ('transformer.h.17.mlp.dense_h_to_4h.weight.84', -59.532562255859375),\n",
       " ('transformer.h.19.mlp.dense_h_to_4h.weight.391', -59.306480407714844),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2705',\n",
       "  -59.095855712890625),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2769',\n",
       "  -59.095848083496094),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2353',\n",
       "  -58.856712341308594),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2417',\n",
       "  -58.856712341308594),\n",
       " ('transformer.h.18.mlp.dense_h_to_4h.weight.84', -58.41749954223633),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2936',\n",
       "  -57.9844856262207),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.3000',\n",
       "  -57.9844856262207),\n",
       " ('transformer.h.17.self_attention.query_key_value_fused_output.weight.1783',\n",
       "  -57.88068389892578),\n",
       " ('transformer.h.17.self_attention.query_key_value_fused_output.weight.1847',\n",
       "  -57.88067626953125),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2347',\n",
       "  -57.4472541809082),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2411',\n",
       "  -57.44724655151367),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2894',\n",
       "  -57.4432258605957),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2958',\n",
       "  -57.4432258605957),\n",
       " ('transformer.h.17.self_attention.query_key_value_fused_output.weight.1758',\n",
       "  -57.20537567138672),\n",
       " ('transformer.h.17.self_attention.query_key_value_fused_output.weight.1822',\n",
       "  -57.20537185668945),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2426',\n",
       "  -57.07352828979492),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2362',\n",
       "  -57.073524475097656),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2943',\n",
       "  -57.0576057434082),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.3007',\n",
       "  -57.0576057434082),\n",
       " ('transformer.h.16.self_attention.query_key_value_fused_output.weight.1853',\n",
       "  -56.904090881347656),\n",
       " ('transformer.h.16.self_attention.query_key_value_fused_output.weight.1789',\n",
       "  -56.904083251953125),\n",
       " ('transformer.h.18.mlp.dense_h_to_4h.weight.35', -56.88310623168945),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.1257',\n",
       "  -56.878211975097656),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.1193',\n",
       "  -56.878204345703125),\n",
       " ('transformer.h.23.self_attention.query_key_value_fused_output.weight.2185',\n",
       "  -56.84088134765625),\n",
       " ('transformer.h.23.self_attention.query_key_value_fused_output.weight.2121',\n",
       "  -56.84087371826172),\n",
       " ('transformer.h.10.self_attention.query_key_value_fused_output.weight.1956',\n",
       "  -56.78551483154297),\n",
       " ('transformer.h.10.self_attention.query_key_value_fused_output.weight.2020',\n",
       "  -56.78550338745117),\n",
       " ('transformer.h.17.self_attention.query_key_value_fused_output.weight.1015',\n",
       "  -56.634681701660156),\n",
       " ('transformer.h.17.self_attention.query_key_value_fused_output.weight.1079',\n",
       "  -56.634674072265625),\n",
       " ('transformer.h.19.mlp.dense_h_to_4h.weight.84', -56.456298828125),\n",
       " ('transformer.h.23.self_attention.query_key_value_fused_output.weight.2118',\n",
       "  -56.26091384887695),\n",
       " ('transformer.h.23.self_attention.query_key_value_fused_output.weight.2182',\n",
       "  -56.26091384887695),\n",
       " ('transformer.h.0.self_attention.query_key_value_fused_output.weight.1537',\n",
       "  -56.19425582885742),\n",
       " ('transformer.h.0.self_attention.query_key_value_fused_output.weight.1601',\n",
       "  -56.194252014160156),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.2308',\n",
       "  -56.158363342285156),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.2372',\n",
       "  -56.158363342285156),\n",
       " ('transformer.h.6.mlp.dense_h_to_4h.weight.12', -56.0640983581543),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2692',\n",
       "  -56.01493453979492),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2756',\n",
       "  -56.01493453979492),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.2522',\n",
       "  -55.88917922973633),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.2586',\n",
       "  -55.88917922973633),\n",
       " ('transformer.h.23.self_attention.query_key_value_fused_output.weight.2553',\n",
       "  -55.65765380859375),\n",
       " ('transformer.h.23.self_attention.query_key_value_fused_output.weight.2617',\n",
       "  -55.65765380859375),\n",
       " ('transformer.h.15.self_attention.query_key_value_fused_output.weight.2342',\n",
       "  -55.550086975097656),\n",
       " ('transformer.h.15.self_attention.query_key_value_fused_output.weight.2406',\n",
       "  -55.550086975097656),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.438',\n",
       "  -55.30279541015625),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.502',\n",
       "  -55.30279541015625),\n",
       " ('transformer.h.9.self_attention.query_key_value_fused_output.weight.2788',\n",
       "  -55.205352783203125),\n",
       " ('transformer.h.9.self_attention.query_key_value_fused_output.weight.2724',\n",
       "  -55.20534133911133),\n",
       " ('transformer.h.15.mlp.dense_h_to_4h.weight.494', -55.13520431518555),\n",
       " ('transformer.h.13.self_attention.query_key_value_fused_output.weight.1959',\n",
       "  -55.1004524230957),\n",
       " ('transformer.h.13.self_attention.query_key_value_fused_output.weight.2023',\n",
       "  -55.10044479370117),\n",
       " ('transformer.h.23.mlp.dense_h_to_4h.weight.276', -55.04045486450195),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.2807',\n",
       "  -54.95139694213867),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.2743',\n",
       "  -54.951393127441406),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2516',\n",
       "  -54.59186935424805),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2580',\n",
       "  -54.59186935424805),\n",
       " ('transformer.h.3.self_attention.query_key_value_fused_output.weight.1730',\n",
       "  -54.58833694458008),\n",
       " ('transformer.h.3.self_attention.query_key_value_fused_output.weight.1794',\n",
       "  -54.58833694458008),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.1732',\n",
       "  -54.56675338745117),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.1796',\n",
       "  -54.56675338745117),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2557',\n",
       "  -54.518165588378906),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2621',\n",
       "  -54.518157958984375),\n",
       " ('transformer.h.6.self_attention.query_key_value_fused_output.weight.1545',\n",
       "  -54.49742889404297),\n",
       " ('transformer.h.6.self_attention.query_key_value_fused_output.weight.1609',\n",
       "  -54.49742889404297),\n",
       " ('transformer.h.9.self_attention.query_key_value_fused_output.weight.2750',\n",
       "  -54.17667770385742),\n",
       " ('transformer.h.9.self_attention.query_key_value_fused_output.weight.2814',\n",
       "  -54.17667770385742),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.2509',\n",
       "  -54.10615158081055),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.2573',\n",
       "  -54.10615158081055),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2710',\n",
       "  -54.02827835083008),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2774',\n",
       "  -54.02827835083008),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2694',\n",
       "  -53.83127975463867),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2758',\n",
       "  -53.83127975463867),\n",
       " ('transformer.h.12.mlp.dense_h_to_4h.weight.996', -53.544036865234375),\n",
       " ('transformer.h.6.self_attention.query_key_value_fused_output.weight.2130',\n",
       "  -53.10606002807617),\n",
       " ('transformer.h.6.self_attention.query_key_value_fused_output.weight.2194',\n",
       "  -53.10606002807617),\n",
       " ('transformer.h.19.self_attention.query_key_value.weight.728',\n",
       "  -52.805015563964844),\n",
       " ('transformer.h.5.self_attention.query_key_value_fused_output.weight.2406',\n",
       "  -52.583213806152344),\n",
       " ('transformer.h.5.self_attention.query_key_value_fused_output.weight.2342',\n",
       "  -52.58320999145508),\n",
       " ('transformer.h.12.self_attention.query_key_value_fused_output.weight.1847',\n",
       "  -52.51691436767578),\n",
       " ('transformer.h.12.self_attention.query_key_value_fused_output.weight.1783',\n",
       "  -52.51690673828125),\n",
       " ('transformer.h.16.mlp.dense_h_to_4h.weight.996', -52.423683166503906),\n",
       " ('transformer.h.13.self_attention.query_key_value_fused_output.weight.2328',\n",
       "  -52.31775665283203),\n",
       " ('transformer.h.13.self_attention.query_key_value_fused_output.weight.2392',\n",
       "  -52.31775665283203),\n",
       " ('transformer.h.10.self_attention.query_key_value_fused_output.weight.1572',\n",
       "  -52.14399337768555),\n",
       " ('transformer.h.10.self_attention.query_key_value_fused_output.weight.1636',\n",
       "  -52.14398956298828),\n",
       " ('transformer.h.3.self_attention.query_key_value_fused_output.weight.1778',\n",
       "  -52.11395263671875),\n",
       " ('transformer.h.3.self_attention.query_key_value_fused_output.weight.1842',\n",
       "  -52.11395263671875),\n",
       " ('transformer.h.0.mlp.dense_h_to_4h.weight.544', -52.0384521484375),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2133',\n",
       "  -52.024452209472656),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2197',\n",
       "  -52.024452209472656),\n",
       " ('transformer.h.5.self_attention.query_key_value_fused_output.weight.2346',\n",
       "  -51.729026794433594),\n",
       " ('transformer.h.5.self_attention.query_key_value_fused_output.weight.2410',\n",
       "  -51.729026794433594),\n",
       " ('transformer.h.20.mlp.dense_h_to_4h.weight.996', -51.72246170043945),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2380',\n",
       "  -51.6562385559082),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2316',\n",
       "  -51.65623092651367),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.1634',\n",
       "  -51.49555587768555),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.1570',\n",
       "  -51.49555206298828),\n",
       " ('transformer.h.12.mlp.dense_h_to_4h.weight.84', -51.28641891479492),\n",
       " ('transformer.h.11.self_attention.query_key_value_fused_output.weight.2998',\n",
       "  -51.128074645996094),\n",
       " ('transformer.h.11.self_attention.query_key_value_fused_output.weight.2934',\n",
       "  -51.12807083129883),\n",
       " ('transformer.h.8.self_attention.query_key_value_fused_output.weight.2806',\n",
       "  -50.47538375854492),\n",
       " ('transformer.h.8.self_attention.query_key_value_fused_output.weight.2742',\n",
       "  -50.475379943847656),\n",
       " ('transformer.h.15.self_attention.query_key_value_fused_output.weight.2028',\n",
       "  -50.459232330322266),\n",
       " ('transformer.h.15.self_attention.query_key_value_fused_output.weight.1964',\n",
       "  -50.459228515625),\n",
       " ('transformer.h.11.self_attention.query_key_value_fused_output.weight.2328',\n",
       "  -50.39599609375),\n",
       " ('transformer.h.11.self_attention.query_key_value_fused_output.weight.2392',\n",
       "  -50.395992279052734),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2734',\n",
       "  -50.204872131347656),\n",
       " ('transformer.h.1.self_attention.query_key_value_fused_output.weight.2798',\n",
       "  -50.204872131347656),\n",
       " ('transformer.h.16.self_attention.query_key_value_fused_output.weight.1757',\n",
       "  -50.14875411987305),\n",
       " ('transformer.h.16.self_attention.query_key_value_fused_output.weight.1821',\n",
       "  -50.14875411987305),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2933',\n",
       "  -49.94672775268555),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2997',\n",
       "  -49.94672393798828),\n",
       " ('transformer.h.17.self_attention.query_key_value_fused_output.weight.1770',\n",
       "  -49.9302864074707),\n",
       " ('transformer.h.17.self_attention.query_key_value_fused_output.weight.1834',\n",
       "  -49.93028259277344),\n",
       " ('transformer.h.2.mlp.dense_h_to_4h.weight.269', -49.817176818847656),\n",
       " ('transformer.h.10.mlp.dense_h_to_4h.weight.785', -49.78860855102539),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.2333',\n",
       "  -49.62149429321289),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.2397',\n",
       "  -49.621490478515625),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.1596',\n",
       "  -49.531280517578125),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.1660',\n",
       "  -49.53127670288086),\n",
       " ('transformer.h.16.self_attention.query_key_value_fused_output.weight.1773',\n",
       "  -49.36494445800781),\n",
       " ('transformer.h.16.self_attention.query_key_value_fused_output.weight.1837',\n",
       "  -49.36494064331055),\n",
       " ('transformer.h.23.self_attention.query_key_value_fused_output.weight.2127',\n",
       "  -49.34369659423828),\n",
       " ('transformer.h.23.self_attention.query_key_value_fused_output.weight.2191',\n",
       "  -49.343692779541016),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.1936',\n",
       "  -49.211219787597656),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.2000',\n",
       "  -49.211219787597656),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2962',\n",
       "  -49.1673698425293),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2898',\n",
       "  -49.16736602783203),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2037',\n",
       "  -49.07042694091797),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1973',\n",
       "  -49.0704231262207),\n",
       " ('transformer.h.6.self_attention.query_key_value_fused_output.weight.1958',\n",
       "  -48.9265022277832),\n",
       " ('transformer.h.6.self_attention.query_key_value_fused_output.weight.2022',\n",
       "  -48.92649841308594),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2795',\n",
       "  -48.91140365600586),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2731',\n",
       "  -48.911399841308594),\n",
       " ('transformer.h.5.self_attention.query_key_value_fused_output.weight.2409',\n",
       "  -48.874732971191406),\n",
       " ('transformer.h.5.self_attention.query_key_value_fused_output.weight.2345',\n",
       "  -48.87472915649414),\n",
       " ('transformer.h.17.mlp.dense_h_to_4h.weight.391', -48.60065460205078),\n",
       " ('transformer.h.9.self_attention.query_key_value_fused_output.weight.2313',\n",
       "  -48.591209411621094),\n",
       " ('transformer.h.9.self_attention.query_key_value_fused_output.weight.2377',\n",
       "  -48.59120559692383),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.794',\n",
       "  -48.54978561401367),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.858',\n",
       "  -48.549781799316406),\n",
       " ('transformer.h.17.self_attention.query_key_value_fused_output.weight.1153',\n",
       "  -48.38310623168945),\n",
       " ('transformer.h.17.self_attention.query_key_value_fused_output.weight.1217',\n",
       "  -48.38310241699219),\n",
       " ('transformer.h.22.mlp.dense_h_to_4h.weight.557', -47.92449951171875),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2531',\n",
       "  -47.76988220214844),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2595',\n",
       "  -47.76988220214844),\n",
       " ('transformer.h.3.self_attention.query_key_value_fused_output.weight.1391',\n",
       "  -47.52405548095703),\n",
       " ('transformer.h.3.self_attention.query_key_value_fused_output.weight.1455',\n",
       "  -47.52405548095703),\n",
       " ('transformer.h.22.mlp.dense_h_to_4h.weight.276', -47.150970458984375),\n",
       " ('transformer.h.9.self_attention.query_key_value_fused_output.weight.2733',\n",
       "  -46.93254089355469),\n",
       " ('transformer.h.9.self_attention.query_key_value_fused_output.weight.2797',\n",
       "  -46.932533264160156),\n",
       " ('transformer.h.12.mlp.dense_h_to_4h.weight.35', -46.84817123413086),\n",
       " ('transformer.h.14.mlp.dense_h_to_4h.weight.785', -46.70429229736328),\n",
       " ('transformer.h.11.self_attention.query_key_value_fused_output.weight.2926',\n",
       "  -46.48961639404297),\n",
       " ('transformer.h.11.self_attention.query_key_value_fused_output.weight.2990',\n",
       "  -46.48961639404297),\n",
       " ('transformer.h.2.mlp.dense_h_to_4h.weight.35', -46.48657989501953),\n",
       " ('transformer.h.11.self_attention.query_key_value_fused_output.weight.2741',\n",
       "  -46.21484375),\n",
       " ('transformer.h.11.self_attention.query_key_value_fused_output.weight.2805',\n",
       "  -46.21483612060547),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1921',\n",
       "  -46.21074676513672),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.1985',\n",
       "  -46.21074295043945),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.1467',\n",
       "  -45.99235153198242),\n",
       " ('transformer.h.18.self_attention.query_key_value_fused_output.weight.1403',\n",
       "  -45.992347717285156),\n",
       " ('transformer.h.12.mlp.dense_h_to_4h.weight.544', -45.9078369140625),\n",
       " ('transformer.h.10.mlp.dense_h_to_4h.weight.494', -45.891929626464844),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2896',\n",
       "  -45.858619689941406),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2960',\n",
       "  -45.858619689941406),\n",
       " ('transformer.h.8.self_attention.query_key_value_fused_output.weight.1189',\n",
       "  -45.782325744628906),\n",
       " ('transformer.h.8.self_attention.query_key_value_fused_output.weight.1253',\n",
       "  -45.782325744628906),\n",
       " ('transformer.h.6.self_attention.query_key_value_fused_output.weight.1992',\n",
       "  -45.62782287597656),\n",
       " ('transformer.h.6.self_attention.query_key_value_fused_output.weight.1928',\n",
       "  -45.6278190612793),\n",
       " ('transformer.h.3.self_attention.query_key_value_fused_output.weight.2358',\n",
       "  -45.464927673339844),\n",
       " ('transformer.h.3.self_attention.query_key_value_fused_output.weight.2422',\n",
       "  -45.46492385864258),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2130',\n",
       "  -45.25304412841797),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2194',\n",
       "  -45.25304412841797),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2348',\n",
       "  -45.12198257446289),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2412',\n",
       "  -45.121978759765625),\n",
       " ('transformer.h.23.mlp.dense_h_to_4h.weight.505', -45.096012115478516),\n",
       " ('transformer.h.16.mlp.dense_h_to_4h.weight.84', -45.04230499267578),\n",
       " ('transformer.h.23.mlp.dense_h_to_4h.weight.76', -44.93257522583008),\n",
       " ('transformer.h.11.self_attention.query_key_value_fused_output.weight.2327',\n",
       "  -44.65846633911133),\n",
       " ('transformer.h.11.self_attention.query_key_value_fused_output.weight.2391',\n",
       "  -44.65846252441406),\n",
       " ('transformer.h.8.self_attention.query_key_value_fused_output.weight.2789',\n",
       "  -44.51094436645508),\n",
       " ('transformer.h.8.self_attention.query_key_value_fused_output.weight.2725',\n",
       "  -44.51093673706055),\n",
       " ('transformer.h.11.mlp.dense_h_to_4h.weight.785', -44.47527313232422),\n",
       " ('transformer.h.9.self_attention.query_key_value_fused_output.weight.2745',\n",
       "  -44.470947265625),\n",
       " ('transformer.h.9.self_attention.query_key_value_fused_output.weight.2809',\n",
       "  -44.470947265625),\n",
       " ('transformer.h.12.self_attention.query_key_value_fused_output.weight.2730',\n",
       "  -44.21300506591797),\n",
       " ('transformer.h.12.self_attention.query_key_value_fused_output.weight.2794',\n",
       "  -44.2130012512207),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2408',\n",
       "  -44.17107391357422),\n",
       " ('transformer.h.21.self_attention.query_key_value_fused_output.weight.2344',\n",
       "  -44.17106628417969),\n",
       " ('transformer.h.7.self_attention.query_key_value_fused_output.weight.2883',\n",
       "  -44.1396369934082),\n",
       " ('transformer.h.7.self_attention.query_key_value_fused_output.weight.2947',\n",
       "  -44.13962936401367),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2578',\n",
       "  -44.002777099609375),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2514',\n",
       "  -44.00277328491211),\n",
       " ('transformer.h.20.mlp.dense_h_to_4h.weight.76', -43.938812255859375),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2910',\n",
       "  -43.623085021972656),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.2974',\n",
       "  -43.623085021972656),\n",
       " ('transformer.h.8.self_attention.query_key_value_fused_output.weight.2203',\n",
       "  -43.59428405761719),\n",
       " ('transformer.h.8.self_attention.query_key_value_fused_output.weight.2139',\n",
       "  -43.59428024291992),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2323',\n",
       "  -43.413421630859375),\n",
       " ('transformer.h.20.self_attention.query_key_value_fused_output.weight.2387',\n",
       "  -43.413421630859375),\n",
       " ('transformer.h.15.self_attention.query_key_value_fused_output.weight.1016',\n",
       "  -43.37000274658203),\n",
       " ('transformer.h.15.self_attention.query_key_value_fused_output.weight.1080',\n",
       "  -43.37000274658203),\n",
       " ('transformer.h.10.mlp.dense_h_to_4h.weight.544', -43.28804397583008),\n",
       " ('transformer.h.10.self_attention.query_key_value_fused_output.weight.2965',\n",
       "  -43.17342758178711),\n",
       " ('transformer.h.10.self_attention.query_key_value_fused_output.weight.2901',\n",
       "  -43.173423767089844),\n",
       " ('transformer.h.6.self_attention.query_key_value_fused_output.weight.1741',\n",
       "  -43.16185760498047),\n",
       " ('transformer.h.6.self_attention.query_key_value_fused_output.weight.1805',\n",
       "  -43.16185760498047),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2197',\n",
       "  -43.10246276855469),\n",
       " ('transformer.h.22.self_attention.query_key_value_fused_output.weight.2133',\n",
       "  -43.10245895385742),\n",
       " ('transformer.h.6.self_attention.query_key_value_fused_output.weight.1770',\n",
       "  -43.02550506591797),\n",
       " ('transformer.h.6.self_attention.query_key_value_fused_output.weight.1834',\n",
       "  -43.02550506591797),\n",
       " ('transformer.h.9.self_attention.query_key_value_fused_output.weight.2720',\n",
       "  -42.982261657714844),\n",
       " ('transformer.h.9.self_attention.query_key_value_fused_output.weight.2784',\n",
       "  -42.982261657714844),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.1783',\n",
       "  -42.975860595703125),\n",
       " ('transformer.h.19.self_attention.query_key_value_fused_output.weight.1847',\n",
       "  -42.975860595703125),\n",
       " ('transformer.h.0.self_attention.query_key_value_fused_output.weight.2135',\n",
       "  -42.944313049316406),\n",
       " ('transformer.h.0.self_attention.query_key_value_fused_output.weight.2199',\n",
       "  -42.944313049316406),\n",
       " ('transformer.h.17.self_attention.query_key_value_fused_output.weight.1013',\n",
       "  -42.843482971191406),\n",
       " ('transformer.h.17.self_attention.query_key_value_fused_output.weight.1077',\n",
       "  -42.843482971191406),\n",
       " ('transformer.h.7.self_attention.query_key_value_fused_output.weight.1578',\n",
       "  -42.5186882019043),\n",
       " ('transformer.h.7.self_attention.query_key_value_fused_output.weight.1642',\n",
       "  -42.5186882019043),\n",
       " ...]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nodes_to_prune"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "05bdc349-a599-4c4d-ac35-c6699946a34b",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "transformer.word_embeddings.weight torch.Size([250880, 1024])\n",
      "transformer.word_embeddings_layernorm.weight torch.Size([1024])\n",
      "transformer.h.0.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.0.self_attention.query_key_value.weight torch.Size([3072, 1024])\n",
      "transformer.h.0.self_attention.dense.weight torch.Size([1024, 1024])\n",
      "transformer.h.0.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.0.mlp.dense_h_to_4h.weight torch.Size([4096, 1024])\n",
      "transformer.h.0.mlp.dense_4h_to_h.weight torch.Size([1024, 4096])\n",
      "transformer.h.1.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.1.self_attention.query_key_value.weight torch.Size([3072, 1024])\n",
      "transformer.h.1.self_attention.dense.weight torch.Size([1024, 1024])\n",
      "transformer.h.1.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.1.mlp.dense_h_to_4h.weight torch.Size([4096, 1024])\n",
      "transformer.h.1.mlp.dense_4h_to_h.weight torch.Size([1024, 4096])\n",
      "transformer.h.2.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.2.self_attention.query_key_value.weight torch.Size([3072, 1024])\n",
      "transformer.h.2.self_attention.dense.weight torch.Size([1024, 1024])\n",
      "transformer.h.2.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.2.mlp.dense_h_to_4h.weight torch.Size([4096, 1024])\n",
      "transformer.h.2.mlp.dense_4h_to_h.weight torch.Size([1024, 4096])\n",
      "transformer.h.3.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.3.self_attention.query_key_value.weight torch.Size([3072, 1024])\n",
      "transformer.h.3.self_attention.dense.weight torch.Size([1024, 1024])\n",
      "transformer.h.3.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.3.mlp.dense_h_to_4h.weight torch.Size([4096, 1024])\n",
      "transformer.h.3.mlp.dense_4h_to_h.weight torch.Size([1024, 4096])\n",
      "transformer.h.4.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.4.self_attention.query_key_value.weight torch.Size([3072, 1024])\n",
      "transformer.h.4.self_attention.dense.weight torch.Size([1024, 1024])\n",
      "transformer.h.4.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.4.mlp.dense_h_to_4h.weight torch.Size([4096, 1024])\n",
      "transformer.h.4.mlp.dense_4h_to_h.weight torch.Size([1024, 4096])\n",
      "transformer.h.5.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.5.self_attention.query_key_value.weight torch.Size([3072, 1024])\n",
      "transformer.h.5.self_attention.dense.weight torch.Size([1024, 1024])\n",
      "transformer.h.5.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.5.mlp.dense_h_to_4h.weight torch.Size([4096, 1024])\n",
      "transformer.h.5.mlp.dense_4h_to_h.weight torch.Size([1024, 4096])\n",
      "transformer.h.6.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.6.self_attention.query_key_value.weight torch.Size([3072, 1024])\n",
      "transformer.h.6.self_attention.dense.weight torch.Size([1024, 1024])\n",
      "transformer.h.6.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.6.mlp.dense_h_to_4h.weight torch.Size([4096, 1024])\n",
      "transformer.h.6.mlp.dense_4h_to_h.weight torch.Size([1024, 4096])\n",
      "transformer.h.7.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.7.self_attention.query_key_value.weight torch.Size([3072, 1024])\n",
      "transformer.h.7.self_attention.dense.weight torch.Size([1024, 1024])\n",
      "transformer.h.7.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.7.mlp.dense_h_to_4h.weight torch.Size([4096, 1024])\n",
      "transformer.h.7.mlp.dense_4h_to_h.weight torch.Size([1024, 4096])\n",
      "transformer.h.8.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.8.self_attention.query_key_value.weight torch.Size([3072, 1024])\n",
      "transformer.h.8.self_attention.dense.weight torch.Size([1024, 1024])\n",
      "transformer.h.8.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.8.mlp.dense_h_to_4h.weight torch.Size([4096, 1024])\n",
      "transformer.h.8.mlp.dense_4h_to_h.weight torch.Size([1024, 4096])\n",
      "transformer.h.9.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.9.self_attention.query_key_value.weight torch.Size([3072, 1024])\n",
      "transformer.h.9.self_attention.dense.weight torch.Size([1024, 1024])\n",
      "transformer.h.9.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.9.mlp.dense_h_to_4h.weight torch.Size([4096, 1024])\n",
      "transformer.h.9.mlp.dense_4h_to_h.weight torch.Size([1024, 4096])\n",
      "transformer.h.10.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.10.self_attention.query_key_value.weight torch.Size([3072, 1024])\n",
      "transformer.h.10.self_attention.dense.weight torch.Size([1024, 1024])\n",
      "transformer.h.10.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.10.mlp.dense_h_to_4h.weight torch.Size([4096, 1024])\n",
      "transformer.h.10.mlp.dense_4h_to_h.weight torch.Size([1024, 4096])\n",
      "transformer.h.11.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.11.self_attention.query_key_value.weight torch.Size([3072, 1024])\n",
      "transformer.h.11.self_attention.dense.weight torch.Size([1024, 1024])\n",
      "transformer.h.11.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.11.mlp.dense_h_to_4h.weight torch.Size([4096, 1024])\n",
      "transformer.h.11.mlp.dense_4h_to_h.weight torch.Size([1024, 4096])\n",
      "transformer.h.12.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.12.self_attention.query_key_value.weight torch.Size([3072, 1024])\n",
      "transformer.h.12.self_attention.dense.weight torch.Size([1024, 1024])\n",
      "transformer.h.12.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.12.mlp.dense_h_to_4h.weight torch.Size([4096, 1024])\n",
      "transformer.h.12.mlp.dense_4h_to_h.weight torch.Size([1024, 4096])\n",
      "transformer.h.13.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.13.self_attention.query_key_value.weight torch.Size([3072, 1024])\n",
      "transformer.h.13.self_attention.dense.weight torch.Size([1024, 1024])\n",
      "transformer.h.13.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.13.mlp.dense_h_to_4h.weight torch.Size([4096, 1024])\n",
      "transformer.h.13.mlp.dense_4h_to_h.weight torch.Size([1024, 4096])\n",
      "transformer.h.14.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.14.self_attention.query_key_value.weight torch.Size([3072, 1024])\n",
      "transformer.h.14.self_attention.dense.weight torch.Size([1024, 1024])\n",
      "transformer.h.14.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.14.mlp.dense_h_to_4h.weight torch.Size([4096, 1024])\n",
      "transformer.h.14.mlp.dense_4h_to_h.weight torch.Size([1024, 4096])\n",
      "transformer.h.15.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.15.self_attention.query_key_value.weight torch.Size([3072, 1024])\n",
      "transformer.h.15.self_attention.dense.weight torch.Size([1024, 1024])\n",
      "transformer.h.15.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.15.mlp.dense_h_to_4h.weight torch.Size([4096, 1024])\n",
      "transformer.h.15.mlp.dense_4h_to_h.weight torch.Size([1024, 4096])\n",
      "transformer.h.16.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.16.self_attention.query_key_value.weight torch.Size([3072, 1024])\n",
      "transformer.h.16.self_attention.dense.weight torch.Size([1024, 1024])\n",
      "transformer.h.16.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.16.mlp.dense_h_to_4h.weight torch.Size([4096, 1024])\n",
      "transformer.h.16.mlp.dense_4h_to_h.weight torch.Size([1024, 4096])\n",
      "transformer.h.17.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.17.self_attention.query_key_value.weight torch.Size([3072, 1024])\n",
      "transformer.h.17.self_attention.dense.weight torch.Size([1024, 1024])\n",
      "transformer.h.17.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.17.mlp.dense_h_to_4h.weight torch.Size([4096, 1024])\n",
      "transformer.h.17.mlp.dense_4h_to_h.weight torch.Size([1024, 4096])\n",
      "transformer.h.18.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.18.self_attention.query_key_value.weight torch.Size([3072, 1024])\n",
      "transformer.h.18.self_attention.dense.weight torch.Size([1024, 1024])\n",
      "transformer.h.18.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.18.mlp.dense_h_to_4h.weight torch.Size([4096, 1024])\n",
      "transformer.h.18.mlp.dense_4h_to_h.weight torch.Size([1024, 4096])\n",
      "transformer.h.19.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.19.self_attention.query_key_value.weight torch.Size([3072, 1024])\n",
      "transformer.h.19.self_attention.dense.weight torch.Size([1024, 1024])\n",
      "transformer.h.19.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.19.mlp.dense_h_to_4h.weight torch.Size([4096, 1024])\n",
      "transformer.h.19.mlp.dense_4h_to_h.weight torch.Size([1024, 4096])\n",
      "transformer.h.20.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.20.self_attention.query_key_value.weight torch.Size([3072, 1024])\n",
      "transformer.h.20.self_attention.dense.weight torch.Size([1024, 1024])\n",
      "transformer.h.20.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.20.mlp.dense_h_to_4h.weight torch.Size([4096, 1024])\n",
      "transformer.h.20.mlp.dense_4h_to_h.weight torch.Size([1024, 4096])\n",
      "transformer.h.21.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.21.self_attention.query_key_value.weight torch.Size([3072, 1024])\n",
      "transformer.h.21.self_attention.dense.weight torch.Size([1024, 1024])\n",
      "transformer.h.21.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.21.mlp.dense_h_to_4h.weight torch.Size([4096, 1024])\n",
      "transformer.h.21.mlp.dense_4h_to_h.weight torch.Size([1024, 4096])\n",
      "transformer.h.22.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.22.self_attention.query_key_value.weight torch.Size([3072, 1024])\n",
      "transformer.h.22.self_attention.dense.weight torch.Size([1024, 1024])\n",
      "transformer.h.22.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.22.mlp.dense_h_to_4h.weight torch.Size([4096, 1024])\n",
      "transformer.h.22.mlp.dense_4h_to_h.weight torch.Size([1024, 4096])\n",
      "transformer.h.23.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.23.self_attention.query_key_value.weight torch.Size([3072, 1024])\n",
      "transformer.h.23.self_attention.dense.weight torch.Size([1024, 1024])\n",
      "transformer.h.23.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.23.mlp.dense_h_to_4h.weight torch.Size([4096, 1024])\n",
      "transformer.h.23.mlp.dense_4h_to_h.weight torch.Size([1024, 4096])\n",
      "transformer.ln_f.weight torch.Size([1024])\n"
     ]
    }
   ],
   "source": [
    "for name, param in model.named_parameters():\n",
    "    if \"bias\" in name:\n",
    "        continue\n",
    "\n",
    "    print(name, param.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fdff57eb-db59-4261-ac7d-47216b94cafe",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Pruning on a Layer Scale"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "79727b09-bd7b-48ed-b8fa-7090a4bec31e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1caca427-1a81-44e6-876a-f1b2b6cef5f6",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tap",
   "language": "python",
   "name": "tap"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}