pruning-strategy.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "75ec25fe-bed3-43a9-94e8-22413302a7a0",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/homebrew/lib/python3.10/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import numpy as np\n",
    "import pickle as pkl\n",
    "\n",
    "from collections import OrderedDict\n",
    "from node_attribution.gradient_node_attribution import get_attributions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "dd5f02b6-a727-47ed-baf0-14ae880caa13",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "human_filtered_pairs = pkl.load(open(\"44_human_filtered_conv_pairs.pkl\", \"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f9cc4ca1-1604-4734-925b-39eed900da8e",
   "metadata": {},
   "outputs": [],
   "source": [
    "calibration_data = human_filtered_pairs[:22]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8efbe0a3-048c-4ca6-a77e-dfe9315161ad",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished loading bigscience/bloom-560m.\n",
      "                    Num Transformer Blocks: 24\n",
      "                    Num Attention Heads: 16\n",
      "                    Head Dim: 64\n",
      "                    Hidden Size: 1024\n",
      "                    Base Model Param Count: 559,214,592\n",
      "                    Total Param Count (w/ LM Head): 816,115,712\n",
      "1\n"
     ]
    }
   ],
   "source": [
    "# Get attributions\n",
    "model_size = \"560m\"\n",
    "avg_contributions, max_contributions, model, model_params = get_attributions(model_size, [calibration_data[0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b9b7a08a-65d7-4158-9e60-fd6839f59c81",
   "metadata": {},
   "outputs": [],
   "source": [
    "#take absolute value of average contributions\n",
    "for layer_name, contribution_tensor in avg_contributions.items():\n",
    "    avg_contributions[layer_name] = torch.abs(avg_contributions[layer_name])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "7840688f-5c3f-4670-b280-0e7a26cef4ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "#pkl.dump(avg_contributions, open(\"avg_contri_560m_22pair_calibration.pkl\", \"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "d57fcb4a-9f44-48d3-9267-5620a8dcfb25",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pkl.dump(max_contributions, open(\"max_contri_3B_22pair_calibration.pkl\", \"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "01fa3686-3fb2-4441-8d0b-2df9ff857ebc",
   "metadata": {},
   "outputs": [],
   "source": [
    "#avg_contributions = pkl.load(open(\"avg_contri_560m_22pair_calibration.pkl\", \"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "289d2a47-0d93-43ad-897b-3ae05dfcce49",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "167764377.6\n"
     ]
    }
   ],
   "source": [
    "prune_percent = 0.30\n",
    "num_params_to_prune = 559214592 * prune_percent\n",
    "\n",
    "head_dim = 64\n",
    "num_heads = 16\n",
    "hidden_size = 1024\n",
    "num_layers = 24\n",
    "\n",
    "print(num_params_to_prune)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "4e880a64-b350-4423-9298-5ef32f86976c",
   "metadata": {},
   "outputs": [],
   "source": [
    "index = 0\n",
    "params_to_index = {}\n",
    "\n",
    "for param_name in model_params.keys():\n",
    "    if \"bias\" not in param_name:\n",
    "        params_to_index[param_name] = index\n",
    "        index += 1\n",
    "    \n",
    "index_to_params = {params_to_index[param_name]: param_name for param_name in params_to_index.keys()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "830cf301-0660-43f4-839c-02016f3bb33f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# layer_shape_map = {}\n",
    "# layer_names, contribution_tensors = zip(*avg_contributions.items())\n",
    "# num_layers = len(layer_names)\n",
    "\n",
    "# for i in range(num_layers):\n",
    "#     layer_name = layer_names[i]\n",
    "#     layer_size = contribution_tensors[i].shape[0]\n",
    "    \n",
    "#     if i != 0:\n",
    "#         next_layer_size = contribution_tensors[i - 1].shape[0]\n",
    "#     else:\n",
    "#         next_layer_size = 250880\n",
    "        \n",
    "    \n",
    "#     if i != (num_layers - 1):\n",
    "#         prev_layer_size = contribution_tensors[i + 1].shape[0]\n",
    "#     else:\n",
    "#         prev_layer_size = 1024\n",
    "        \n",
    "#     layer_shape_map[layer_name] = {\n",
    "#         \"prev_layer_size\": prev_layer_size,\n",
    "#         \"next_layer_size\": next_layer_size,\n",
    "#         \"current_layer_size\": layer_size\n",
    "#     }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "52304b74-b166-4c13-a95f-2eafc0c3a705",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "all_nodes = []\n",
    "\n",
    "for layer_name, contribution_tensor in avg_contributions.items():\n",
    "    for node_id, node in enumerate(contribution_tensor.tolist()):\n",
    "        node_name = f\"{layer_name}.{node_id}\"\n",
    "        all_nodes.append((node_name, node))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "fee0d3a6-0b98-4e18-9940-34b0f3ed7db4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sort all the nodes\n",
    "all_nodes.sort(key = lambda x:x[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "7061e785-c991-4e60-b4c3-80bfb778cf1c",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_layers = []\n",
    "\n",
    "# Figure out which layers have the most to prune\n",
    "for layer_name, contribution_tensor in avg_contributions.items():\n",
    "    \n",
    "    # Don't prune self_attention.dense.weight directly, use value matrix to decide what to prune\n",
    "    # Get average contribution over the whole layer\n",
    "    mean_contribution = torch.mean(contribution_tensor, 0).item()\n",
    "    all_layers.append((layer_name, mean_contribution))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "6568baf2-3382-4e41-a309-e20b827d8830",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_layers.sort(key = lambda x:x[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "8228ab31",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('transformer.h.23.mlp.dense_4h_to_h.weight', 0.00235293572768569),\n",
       " ('transformer.h.9.mlp.dense_4h_to_h.weight', 0.0025348886847496033),\n",
       " ('transformer.h.8.mlp.dense_4h_to_h.weight', 0.002768360311165452),\n",
       " ('transformer.h.7.mlp.dense_4h_to_h.weight', 0.0037689958699047565),\n",
       " ('transformer.h.2.self_attention.value.weight', 0.00436257291585207),\n",
       " ('transformer.h.8.self_attention.value.weight', 0.004458594601601362),\n",
       " ('transformer.h.10.self_attention.value.weight', 0.004510129801928997),\n",
       " ('transformer.h.23.self_attention.value.weight', 0.00453915074467659),\n",
       " ('transformer.h.3.self_attention.value.weight', 0.0045631686225533485),\n",
       " ('transformer.h.0.mlp.dense_4h_to_h.weight', 0.004628546070307493),\n",
       " ('transformer.h.10.mlp.dense_4h_to_h.weight', 0.004696134943515062),\n",
       " ('transformer.h.5.mlp.dense_4h_to_h.weight', 0.004815695807337761),\n",
       " ('transformer.h.1.mlp.dense_4h_to_h.weight', 0.004817564971745014),\n",
       " ('transformer.h.11.mlp.dense_4h_to_h.weight', 0.004893769044429064),\n",
       " ('transformer.h.1.self_attention.value.weight', 0.004914944525808096),\n",
       " ('transformer.h.6.self_attention.value.weight', 0.0049645304679870605),\n",
       " ('transformer.h.4.self_attention.value.weight', 0.005040680058300495),\n",
       " ('transformer.h.5.self_attention.value.weight', 0.0050426749512553215),\n",
       " ('transformer.h.0.self_attention.value.weight', 0.005118215922266245),\n",
       " ('transformer.h.7.self_attention.value.weight', 0.005145685281604528),\n",
       " ('transformer.h.6.mlp.dense_4h_to_h.weight', 0.005209535825997591),\n",
       " ('transformer.h.11.self_attention.value.weight', 0.005315016955137253),\n",
       " ('transformer.h.4.mlp.dense_4h_to_h.weight', 0.005331296939402819),\n",
       " ('transformer.h.2.mlp.dense_4h_to_h.weight', 0.005341458134353161),\n",
       " ('transformer.h.12.self_attention.value.weight', 0.00535980798304081),\n",
       " ('transformer.h.3.mlp.dense_4h_to_h.weight', 0.0054097711108624935),\n",
       " ('transformer.h.13.mlp.dense_4h_to_h.weight', 0.005589435342699289),\n",
       " ('transformer.h.9.self_attention.value.weight', 0.005607145372778177),\n",
       " ('transformer.h.13.self_attention.value.weight', 0.005613858811557293),\n",
       " ('transformer.h.16.self_attention.value.weight', 0.005662544164806604),\n",
       " ('transformer.h.14.mlp.dense_4h_to_h.weight', 0.005698923487216234),\n",
       " ('transformer.h.12.mlp.dense_4h_to_h.weight', 0.005721147637814283),\n",
       " ('transformer.h.15.self_attention.value.weight', 0.005838292185217142),\n",
       " ('transformer.h.17.self_attention.value.weight', 0.005925819277763367),\n",
       " ('transformer.h.15.mlp.dense_4h_to_h.weight', 0.006111299619078636),\n",
       " ('transformer.h.14.self_attention.value.weight', 0.00621975539252162),\n",
       " ('transformer.h.19.self_attention.value.weight', 0.006341368425637484),\n",
       " ('transformer.h.18.self_attention.value.weight', 0.006499454379081726),\n",
       " ('transformer.h.22.mlp.dense_4h_to_h.weight', 0.006672971416264772),\n",
       " ('transformer.h.16.mlp.dense_4h_to_h.weight', 0.006804308854043484),\n",
       " ('transformer.h.17.mlp.dense_4h_to_h.weight', 0.0073576257564127445),\n",
       " ('transformer.h.18.mlp.dense_4h_to_h.weight', 0.008014407940208912),\n",
       " ('transformer.h.20.mlp.dense_4h_to_h.weight', 0.008064473047852516),\n",
       " ('transformer.h.20.self_attention.value.weight', 0.00848489161580801),\n",
       " ('transformer.h.19.mlp.dense_4h_to_h.weight', 0.00848598312586546),\n",
       " ('transformer.h.21.mlp.dense_4h_to_h.weight', 0.00874361116439104),\n",
       " ('transformer.h.21.self_attention.value.weight', 0.009903538972139359),\n",
       " ('transformer.h.22.self_attention.value.weight', 0.0102576008066535)]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "b6f75b8e-7803-464c-b368-21c453ba0725",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "layer_masks = {}\n",
    "num_params_pruned = 0\n",
    "node_num = 0\n",
    "min_nodes = 24\n",
    "min_heads = 4\n",
    "value_dim = 2\n",
    "\n",
    "while num_params_pruned < num_params_to_prune:\n",
    "    lowest_contr_layer_name = all_layers[0][0]               \n",
    "    stop_pruning_layer = False\n",
    "    \n",
    "    if \"self_attention.value.weight\" in lowest_contr_layer_name:\n",
    "        if lowest_contr_layer_name not in layer_masks:\n",
    "            layer_contributions = avg_contributions[lowest_contr_layer_name]\n",
    "            layer_contributions = layer_contributions.view(num_heads, head_dim)\n",
    "            head_contributions = []\n",
    "            \n",
    "            for head in range(num_heads):\n",
    "                head_score = torch.mean(layer_contributions[head])\n",
    "                head_contributions.append(head_score)\n",
    "                \n",
    "            head_contributions = torch.stack(head_contributions)\n",
    "            sorted_heads = torch.argsort(head_contributions)        \n",
    "            mask = torch.zeros_like(layer_contributions)\n",
    "            num_pruned = 0\n",
    "\n",
    "        else:\n",
    "            mask = layer_masks[lowest_contr_layer_name][0]\n",
    "            sorted_heads = layer_masks[lowest_contr_layer_name][1]\n",
    "            num_pruned = layer_masks[lowest_contr_layer_name][2] \n",
    "            \n",
    "            \n",
    "        head_to_prune = sorted_heads[num_pruned]\n",
    "        for i in range(head_dim):\n",
    "            mask[head_to_prune][i] = 1\n",
    "            \n",
    "        heads_left = num_heads - num_pruned - 1\n",
    "        \n",
    "        # Keep from deleting all nodes in a layer\n",
    "        if heads_left > min_heads:\n",
    "            layer_masks[lowest_contr_layer_name] = (mask, sorted_heads, num_pruned + 1)\n",
    "            num_params_pruned += (head_dim * 3 * hidden_size)\n",
    "            num_params_pruned += (head_dim * hidden_size)\n",
    "            node_num += (head_dim * 3)\n",
    "        else:\n",
    "            stop_pruning_layer = True\n",
    "    \n",
    "    else:\n",
    "        # Prune one node at time\n",
    "        if lowest_contr_layer_name not in layer_masks:\n",
    "            layer_contributions = avg_contributions[lowest_contr_layer_name]\n",
    "            mask = torch.zeros_like(layer_contributions)\n",
    "            sorted_contributions = torch.argsort(layer_contributions)\n",
    "            num_pruned = 0\n",
    "\n",
    "        else:\n",
    "            mask = layer_masks[lowest_contr_layer_name][0]\n",
    "            sorted_contributions = layer_masks[lowest_contr_layer_name][1]\n",
    "            num_pruned = layer_masks[lowest_contr_layer_name][2]\n",
    "\n",
    "        index_to_mask = sorted_contributions[num_pruned]\n",
    "        mask[index_to_mask] = 1\n",
    "\n",
    "        nodes_left = torch.numel(mask) - int(torch.sum(mask).item())\n",
    "\n",
    "        # Keep from deleting all nodes in a layer\n",
    "        if nodes_left > min_nodes:\n",
    "            layer_masks[lowest_contr_layer_name] = (mask, sorted_contributions, num_pruned + 1)\n",
    "            num_params_pruned += 2 * hidden_size\n",
    "            node_num += 1\n",
    "        else:\n",
    "            stop_pruning_layer = True\n",
    "\n",
    "    # Apply mask and update the layer mean in \"all_layers\"\n",
    "    if stop_pruning_layer:\n",
    "        new_layer_contr_score = float('inf')\n",
    "    else:\n",
    "        mean_array = np.ma.array(data=avg_contributions[lowest_contr_layer_name], mask=mask)\n",
    "        new_layer_contr_score = mean_array.mean()\n",
    "\n",
    "    all_layers[0] = (lowest_contr_layer_name, new_layer_contr_score)\n",
    "    all_layers.sort(key = lambda x:x[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "2fd89943",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "transformer.h.23.mlp.dense_4h_to_h.weight 3315\n",
      "transformer.h.9.mlp.dense_4h_to_h.weight 3287\n",
      "transformer.h.8.mlp.dense_4h_to_h.weight 3293\n",
      "transformer.h.7.mlp.dense_4h_to_h.weight 2949\n",
      "transformer.h.2.self_attention.value.weight 11\n",
      "transformer.h.8.self_attention.value.weight 11\n",
      "transformer.h.10.self_attention.value.weight 11\n",
      "transformer.h.23.self_attention.value.weight 11\n",
      "transformer.h.3.self_attention.value.weight 11\n",
      "transformer.h.0.mlp.dense_4h_to_h.weight 2662\n",
      "transformer.h.10.mlp.dense_4h_to_h.weight 2560\n",
      "transformer.h.5.mlp.dense_4h_to_h.weight 2656\n",
      "transformer.h.1.mlp.dense_4h_to_h.weight 2698\n",
      "transformer.h.11.mlp.dense_4h_to_h.weight 2569\n",
      "transformer.h.1.self_attention.value.weight 11\n",
      "transformer.h.6.self_attention.value.weight 11\n",
      "transformer.h.4.self_attention.value.weight 11\n",
      "transformer.h.5.self_attention.value.weight 11\n",
      "transformer.h.0.self_attention.value.weight 11\n",
      "transformer.h.7.self_attention.value.weight 11\n",
      "transformer.h.6.mlp.dense_4h_to_h.weight 2521\n",
      "transformer.h.11.self_attention.value.weight 11\n",
      "transformer.h.4.mlp.dense_4h_to_h.weight 2547\n",
      "transformer.h.2.mlp.dense_4h_to_h.weight 2484\n",
      "transformer.h.12.self_attention.value.weight 11\n",
      "transformer.h.3.mlp.dense_4h_to_h.weight 2466\n",
      "transformer.h.13.mlp.dense_4h_to_h.weight 2330\n",
      "transformer.h.9.self_attention.value.weight 11\n",
      "transformer.h.13.self_attention.value.weight 11\n",
      "transformer.h.16.self_attention.value.weight 11\n",
      "transformer.h.14.mlp.dense_4h_to_h.weight 2263\n",
      "transformer.h.12.mlp.dense_4h_to_h.weight 2279\n",
      "transformer.h.15.self_attention.value.weight 11\n",
      "transformer.h.17.self_attention.value.weight 11\n",
      "transformer.h.15.mlp.dense_4h_to_h.weight 1976\n",
      "transformer.h.14.self_attention.value.weight 11\n",
      "transformer.h.19.self_attention.value.weight 11\n",
      "transformer.h.18.self_attention.value.weight 11\n",
      "transformer.h.22.mlp.dense_4h_to_h.weight 1481\n",
      "transformer.h.16.mlp.dense_4h_to_h.weight 1519\n",
      "transformer.h.17.mlp.dense_4h_to_h.weight 1186\n",
      "transformer.h.18.mlp.dense_4h_to_h.weight 831\n",
      "transformer.h.20.mlp.dense_4h_to_h.weight 797\n",
      "transformer.h.20.self_attention.value.weight 5\n",
      "transformer.h.19.mlp.dense_4h_to_h.weight 586\n",
      "transformer.h.21.mlp.dense_4h_to_h.weight 454\n"
     ]
    }
   ],
   "source": [
    "for key, value in layer_masks.items():\n",
    "    print(key, value[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "fc0d48da-8f45-4f78-a502-ed4e413b6799",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Line up weights to prune and weights in the state dict\n",
    "mask_index = 0\n",
    "sorted_weight_index = 1\n",
    "copied_model_params = model_params.copy()\n",
    "pruned_model_params = OrderedDict()\n",
    "\n",
    "for layer_name in copied_model_params:\n",
    "    if layer_name in layer_masks.keys():\n",
    "        if \"self_attention.value.weight\" in layer_name:   \n",
    "            # Prune as input\n",
    "            # Look at value matrix to decide what should be droped in \"self_attention.dense.weight\"\n",
    "            value_reshape_mask = layer_masks[layer_name][mask_index].reshape(num_heads * head_dim)\n",
    "            value_keep_index = torch.tensor([i for i in range(value_reshape_mask.shape[-1]) if value_reshape_mask[i] == 0])\n",
    "\n",
    "            qk_layer_name = layer_name.replace(\"value\", \"query_key\")\n",
    "            qk_keep_index = torch.add(value_keep_index, hidden_size)\n",
    "            qk_keep_index = torch.cat([value_keep_index, qk_keep_index], 0)\n",
    "\n",
    "            pruned_query_key_output_weights = torch.index_select(copied_model_params[qk_layer_name], 0, qk_keep_index)\n",
    "            pruned_value_output_weights = torch.index_select(copied_model_params[layer_name], 0, value_keep_index)\n",
    "\n",
    "            qkv_layer_name = layer_name.replace(\"value\", \"query_key_value\")\n",
    "            pruned_query_key_value_weights = torch.cat((pruned_query_key_output_weights, pruned_value_output_weights), 0)\n",
    "            pruned_model_params[qkv_layer_name] = pruned_query_key_value_weights\n",
    "\n",
    "            # Also do bias terms\n",
    "            qk_bias_name = qk_layer_name.replace(\"weight\", \"bias\")\n",
    "            value_bias_name = layer_name.replace(\"weight\", \"bias\")\n",
    "            qkv_bias_name = qkv_layer_name.replace(\"weight\", \"bias\")\n",
    "            pruned_qk_bias = torch.index_select(copied_model_params[qk_bias_name], 0, qk_keep_index)\n",
    "            pruned_value_bias = torch.index_select(copied_model_params[value_bias_name], 0, value_keep_index)\n",
    "            pruned_query_key_value_bias = torch.cat((pruned_qk_bias, pruned_value_bias), 0)\n",
    "            pruned_model_params[qkv_bias_name] = pruned_query_key_value_bias\n",
    "            \n",
    "            dense_layer_name = layer_name.replace(\"value\", \"dense\")\n",
    "            pruned_dense_input_weights = torch.index_select(copied_model_params[dense_layer_name], -1, value_keep_index)\n",
    "            pruned_model_params[dense_layer_name] = pruned_dense_input_weights\n",
    "\n",
    "        else:\n",
    "            # Prune when nodes are the input\n",
    "            num_nodes_to_drop = int(sum(layer_masks[layer_name][mask_index]).item())\n",
    "            keep_index = torch.sort(layer_masks[layer_name][sorted_weight_index][num_nodes_to_drop:]).values\n",
    "            pruned_input_weights = torch.index_select(copied_model_params[layer_name], -1, keep_index)\n",
    "            pruned_model_params[layer_name] = pruned_input_weights\n",
    "\n",
    "            # Go to previous layer and prune when nodes are the output\n",
    "            prev_layer_index = params_to_index[layer_name] - 1\n",
    "            prev_layer_name = index_to_params[prev_layer_index]\n",
    "\n",
    "            if \"layernorm\" in prev_layer_name:\n",
    "                pruned_layer_norm_weights = torch.index_select(copied_model_params[prev_layer_name], 0, keep_index)\n",
    "                pruned_model_params[prev_layer_name] = pruned_layer_norm_weights \n",
    "\n",
    "                # Also do bias term\n",
    "                bias_layer_name = prev_layer_name.replace(\"weight\", \"bias\")\n",
    "                pruned_bias_weights = torch.index_select(copied_model_params[bias_layer_name], 0, keep_index)\n",
    "                pruned_model_params[bias_layer_name] = pruned_bias_weights\n",
    "\n",
    "                prev_layer_index = prev_layer_index - 1\n",
    "                prev_layer_name = index_to_params[prev_layer_index]\n",
    "\n",
    "            pruned_output_weights = torch.index_select(copied_model_params[prev_layer_name], 0, keep_index)\n",
    "            pruned_model_params[prev_layer_name] = pruned_output_weights\n",
    "\n",
    "            # Also do bias term\n",
    "            bias_layer_name = prev_layer_name.replace(\"weight\", \"bias\")\n",
    "            pruned_bias_weights = torch.index_select(copied_model_params[bias_layer_name], 0, keep_index)\n",
    "            pruned_model_params[bias_layer_name] = pruned_bias_weights\n",
    "            \n",
    "    elif \"self_attention.value.weight\" in layer_name:\n",
    "        value_weights = copied_model_params[layer_name]\n",
    "        query_key_layer = layer_name.replace(\"value\", \"query_key\")\n",
    "        query_key_weights = copied_model_params[query_key_layer]\n",
    "        \n",
    "        new_layer_name = layer_name.replace(\"value\", \"query_key_value\")\n",
    "        new_layer_weights = torch.cat((query_key_weights, value_weights), 0)\n",
    "        pruned_model_params[new_layer_name] = new_layer_weights\n",
    "        \n",
    "        # Do bias\n",
    "        value_bias_name = layer_name.replace(\"weight\", \"bias\")\n",
    "        query_key_bias_name = query_key_layer.replace(\"weight\", \"bias\")\n",
    "        new_layer_bias_name = new_layer_name.replace(\"weight\", \"bias\")\n",
    "        \n",
    "        value_bias = copied_model_params[value_bias_name]\n",
    "        query_key_bias = copied_model_params[query_key_bias_name]\n",
    "        new_layer_bias = torch.cat((query_key_bias, value_bias), 0)\n",
    "        pruned_model_params[new_layer_bias_name] = new_layer_bias\n",
    "        \n",
    "        # Do dense layer\n",
    "        dense_layer_name = layer_name.replace(\"value\", \"dense\")\n",
    "        pruned_model_params[dense_layer_name] = copied_model_params[dense_layer_name]\n",
    "        \n",
    "    elif \"self_attention.value.bias\" in layer_name:\n",
    "        continue\n",
    "    elif \"self_attention.query_key.weight\" in layer_name:\n",
    "        continue\n",
    "    elif \"self_attention.query_key.bias\" in layer_name:\n",
    "        continue\n",
    "    elif \"self_attention.dense.weight\" in layer_name:\n",
    "        continue\n",
    "    else:\n",
    "        pruned_model_params[layer_name] = copied_model_params[layer_name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "7e3974dc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OrderedDict([('transformer.word_embeddings.weight',\n",
       "              tensor([[-0.0099, -0.0048, -0.0111,  ..., -0.0426,  0.0099,  0.0212],\n",
       "                      [ 0.0048, -0.0127,  0.0138,  ..., -0.0448,  0.0003, -0.0120],\n",
       "                      [ 0.0065,  0.0239,  0.0050,  ..., -0.0431, -0.0067,  0.0137],\n",
       "                      ...,\n",
       "                      [-0.0028, -0.0038, -0.0012,  ..., -0.0252,  0.0013,  0.0012],\n",
       "                      [-0.0028, -0.0038, -0.0012,  ..., -0.0252,  0.0013,  0.0012],\n",
       "                      [-0.0028, -0.0038, -0.0012,  ..., -0.0252,  0.0013,  0.0012]])),\n",
       "             ('transformer.word_embeddings_layernorm.weight',\n",
       "              tensor([0.4409, 0.3167, 0.4749,  ..., 0.0816, 0.2927, 0.6006])),\n",
       "             ('transformer.word_embeddings_layernorm.bias',\n",
       "              tensor([-0.0513,  0.0164,  0.0052,  ...,  0.2412,  0.0072, -0.0292])),\n",
       "             ('transformer.h.0.input_layernorm.weight',\n",
       "              tensor([0.6621, 0.8457, 0.5884,  ..., 3.2090, 0.9180, 0.5586])),\n",
       "             ('transformer.h.0.input_layernorm.bias',\n",
       "              tensor([-0.0662, -0.0798,  0.0445,  ..., -0.1449, -0.0452,  0.0142])),\n",
       "             ('transformer.h.0.self_attention.query_key_value.weight',\n",
       "              tensor([[-8.2932e-03,  1.4915e-02, -9.1400e-03,  ..., -2.5883e-03,\n",
       "                       -9.9640e-03, -1.2276e-02],\n",
       "                      [-8.7891e-03, -1.2581e-02, -5.7297e-03,  ..., -8.8882e-03,\n",
       "                       -1.4351e-02,  4.0169e-03],\n",
       "                      [ 2.7985e-02,  1.1147e-02,  1.5579e-02,  ..., -1.3888e-05,\n",
       "                        9.1629e-03, -9.5444e-03],\n",
       "                      ...,\n",
       "                      [ 7.4730e-03, -6.9962e-03, -1.2009e-02,  ..., -5.8174e-04,\n",
       "                        7.0989e-05, -7.3738e-03],\n",
       "                      [ 2.2812e-02, -5.9624e-03,  1.5427e-02,  ...,  2.8706e-03,\n",
       "                       -1.0281e-03,  1.0826e-02],\n",
       "                      [-4.9248e-03, -8.2245e-03, -2.6550e-03,  ..., -8.4991e-03,\n",
       "                       -1.9409e-02, -1.7303e-02]])),\n",
       "             ('transformer.h.0.self_attention.query_key_value.bias',\n",
       "              tensor([ 6.3934e-03,  3.1395e-03,  1.4191e-02,  2.1896e-02, -8.2626e-03,\n",
       "                       9.9182e-03,  2.5101e-02,  1.0262e-03, -1.0681e-03, -1.8509e-02,\n",
       "                      -1.0414e-02, -2.4155e-02, -1.2390e-02, -1.5602e-02, -1.5160e-02,\n",
       "                       1.9140e-03, -2.4155e-02, -2.9221e-02,  3.0563e-02, -6.3354e-02,\n",
       "                      -1.6525e-02, -7.7896e-03, -1.8204e-02, -2.5043e-03, -4.5562e-04,\n",
       "                       2.1610e-03, -1.2810e-02,  3.4065e-03,  1.6571e-02,  1.2939e-02,\n",
       "                       2.0142e-02,  4.1056e-04,  1.5049e+00, -5.3467e-01, -5.7220e-03,\n",
       "                      -6.4583e-03,  1.5869e-03, -1.0139e-02,  1.6144e-02, -1.3405e-02,\n",
       "                      -1.3931e-02,  2.5269e-02, -1.0536e-02,  1.9073e-02,  1.3771e-02,\n",
       "                       2.9694e-02, -1.6647e-02, -2.9202e-03,  4.8187e-02,  2.5681e-02,\n",
       "                       1.9627e-03,  2.5269e-02, -1.0529e-02, -1.8387e-02, -6.9122e-03,\n",
       "                      -7.3090e-03,  1.1597e-03,  6.8741e-03, -2.2602e-03,  1.4107e-02,\n",
       "                       6.3934e-03, -1.3329e-02, -3.0065e-04,  2.8801e-03,  6.5796e-02,\n",
       "                      -2.1326e-01,  4.2084e-02,  5.1514e-01,  5.4993e-02, -3.8696e-02,\n",
       "                      -2.5928e-01, -2.2620e-01, -3.2251e-01,  1.6333e-01,  2.1027e-02,\n",
       "                       3.5767e-02,  1.0796e-02,  1.2085e-01,  8.9050e-02,  1.5173e-01,\n",
       "                      -3.4766e-01, -1.3220e-01, -3.7537e-02, -1.2103e-01, -1.7944e-01,\n",
       "                      -1.7371e-01, -1.1635e-02, -1.6052e-01, -9.0942e-02, -3.3836e-03,\n",
       "                       5.7812e-01, -4.8447e-03,  2.7588e-01, -3.3081e-02, -2.0715e-01,\n",
       "                      -3.1250e-01,  2.6688e-02,  1.8689e-01,  2.7374e-02,  3.0933e-01,\n",
       "                      -3.1030e-01,  9.2834e-02,  1.2769e-01,  2.7295e-01, -3.2440e-02,\n",
       "                      -5.0439e-01,  1.6968e-02, -1.2866e-01,  2.3254e-02, -1.8692e-03,\n",
       "                      -2.6416e-01,  1.2398e-02, -2.0599e-02,  3.9233e-01,  1.5479e-01,\n",
       "                      -6.6772e-02, -1.6040e-01, -8.1909e-02, -9.9487e-02,  2.5314e-02,\n",
       "                      -1.2067e-01, -1.7114e-01, -3.7671e-01, -1.6724e-02, -4.0497e-02,\n",
       "                       3.4515e-02,  1.0272e-01,  1.6846e-02,  2.4719e-02,  2.9981e-05,\n",
       "                       8.4686e-03,  1.0818e-02, -3.1071e-03, -1.1261e-02, -4.9591e-03,\n",
       "                       1.6541e-02,  7.2021e-03, -2.2171e-02,  9.7656e-03,  1.1225e-03,\n",
       "                       1.4582e-03,  3.9711e-03, -9.5367e-03, -2.0081e-02, -7.9193e-03,\n",
       "                      -5.5725e-02, -3.1830e-02, -5.7030e-03, -6.2294e-03, -4.4098e-02,\n",
       "                       1.1475e-02, -2.2980e-02, -4.4891e-02, -2.4891e-03,  1.2871e-02,\n",
       "                       1.0201e-02, -5.6763e-03, -1.3748e-02, -1.7366e-03,  9.2840e-04,\n",
       "                      -1.5497e-03, -1.9333e-02, -1.2062e-02,  3.3665e-03, -2.0046e-03,\n",
       "                       1.8740e-03,  6.4373e-04,  2.6073e-03,  1.1673e-02, -7.2746e-03,\n",
       "                       9.1476e-03, -1.5274e-02, -5.7316e-04,  1.5793e-02,  8.6746e-03,\n",
       "                       1.9073e-02, -6.0844e-03,  5.3673e-03,  8.4610e-03, -3.6888e-03,\n",
       "                       8.5983e-03,  2.0044e-01, -4.3869e-03, -2.3041e-02,  9.3460e-03,\n",
       "                      -5.7869e-03, -1.2589e-02, -6.4125e-03,  9.8288e-05,  5.4741e-03,\n",
       "                      -7.3090e-03,  1.6068e-02, -1.9531e-02, -2.0374e-01, -7.4768e-02,\n",
       "                      -1.8867e+00, -1.5051e-01,  1.9348e-02, -2.9517e-01, -1.1707e-01,\n",
       "                      -4.1718e-02,  9.3750e-02, -4.4312e-01,  1.7395e-01, -1.0095e-01,\n",
       "                      -3.4485e-02, -1.3696e-01, -1.1694e-01,  3.2562e-02,  2.1802e-01,\n",
       "                      -6.1279e-01, -5.8502e-02, -8.6243e-02,  2.4731e-01, -4.6600e-02,\n",
       "                       2.8760e-01,  1.7624e-02,  1.4929e-01,  1.0791e-01,  5.7831e-02,\n",
       "                       9.4299e-02,  3.6377e-01,  8.1558e-03,  4.4861e-02,  1.6052e-01,\n",
       "                       2.8107e-02, -8.4778e-02,  1.1169e-01, -1.1835e-01, -7.1838e-02,\n",
       "                      -6.1127e-02, -4.0161e-02, -1.3049e-01, -2.6147e-01,  8.5815e-02,\n",
       "                      -1.2805e-01, -6.5918e-02, -4.4800e-02, -9.6008e-02,  9.1553e-03,\n",
       "                      -5.4535e-02,  8.0017e-02, -6.3904e-02, -5.6534e-03, -2.0166e-01,\n",
       "                       1.4185e-01, -1.8984e+00,  5.0812e-02,  2.8174e-01,  1.8359e-01,\n",
       "                      -2.1350e-01, -9.1370e-02, -2.1774e-02,  7.1716e-02,  6.4270e-02,\n",
       "                      -1.0785e-01, -2.2559e-01,  1.2189e-01,  2.0248e-02, -1.8408e-01,\n",
       "                      -6.3324e-03, -1.3538e-01,  3.3301e-01, -8.5510e-02,  1.6553e-01,\n",
       "                       2.3098e-03,  1.0992e-01,  1.7212e-02,  9.6130e-03,  2.1240e-01,\n",
       "                       9.4604e-03,  1.3931e-02, -1.9324e-01, -2.2046e-01, -9.9670e-02,\n",
       "                       1.6980e-01, -8.1055e-02,  1.7480e-01,  1.0977e-03,  2.8125e-01,\n",
       "                       1.3733e-01,  5.3223e-01,  3.6907e-03, -1.0840e-01,  1.6370e-01,\n",
       "                       3.0542e-01, -1.1761e-01,  1.2856e-03,  2.9861e-02,  2.0239e-01,\n",
       "                      -8.5693e-02, -1.1505e-01,  1.2854e-01,  3.7903e-02, -5.3787e-03,\n",
       "                      -9.4299e-02,  2.0557e-01,  2.1130e-01,  9.4238e-01, -1.0553e-01,\n",
       "                      -1.5674e-01, -3.5980e-02,  1.8909e-01,  6.2347e-02,  2.0728e-01,\n",
       "                       9.4177e-02, -2.5488e-01, -1.1084e-01, -1.7012e+00,  6.2622e-02,\n",
       "                       1.0681e-01, -5.3027e-01,  1.8018e-01,  1.5881e-01, -4.0588e-02,\n",
       "                      -1.6028e-01, -2.1692e-01,  5.3596e-03,  2.7759e-01,  7.7393e-02,\n",
       "                      -1.4282e-01,  5.6445e+00,  3.7646e-01, -4.7424e-02,  2.1350e-01,\n",
       "                      -1.4050e-01, -1.1597e-01,  5.5859e-01,  1.8164e-01, -9.8975e-01,\n",
       "                      -2.0386e-01,  4.0234e-01,  6.3867e-01,  4.1260e-01,  7.1411e-03,\n",
       "                      -1.7178e+00, -2.4170e-01, -1.3416e-01,  6.4160e-01, -7.7441e-01,\n",
       "                       8.0811e-02, -7.7783e-01, -8.4521e-01,  3.6450e-01, -2.1927e-02,\n",
       "                       1.2686e+00, -3.8477e-01,  1.5857e-01,  2.2449e-01,  1.4856e-01,\n",
       "                       6.6455e-01,  8.1863e-03, -1.8555e-01,  1.9250e-01,  6.5002e-02,\n",
       "                       1.6232e-03, -3.7036e-01,  4.9194e-01, -6.1670e-01, -8.5878e-04,\n",
       "                      -3.3716e-01,  2.6636e-01,  6.2207e-01, -2.1680e-01, -2.5059e+00,\n",
       "                      -4.1821e-01,  2.4918e-02, -2.6758e-01, -3.6304e-01,  9.7803e-01,\n",
       "                       4.8950e-01,  2.8442e-01,  5.2539e-01, -4.3457e-01,  6.5967e-01,\n",
       "                      -7.2852e-01,  2.2961e-01, -1.9104e-01,  2.0059e+00, -9.2871e-01,\n",
       "                      -7.4365e-01, -2.7490e-01, -7.2998e-01, -4.0088e-01, -4.8889e-02,\n",
       "                      -1.4624e-01, -1.2537e-01,  1.5955e-01, -2.7695e-02, -4.1885e-03,\n",
       "                      -3.3179e-01, -6.2891e-01, -3.6060e-01, -2.8418e-01, -8.8721e-01,\n",
       "                       3.4277e-01,  5.8258e-02,  3.3789e-01, -5.1953e-01, -5.5859e-01,\n",
       "                       1.5511e-02,  4.1895e-01,  5.8936e-01, -3.3569e-01,  1.9824e-01,\n",
       "                      -8.3838e-01, -7.7686e-01, -2.8931e-01,  3.4302e-01,  3.0322e-01,\n",
       "                       4.2114e-02,  1.0980e-01, -1.8701e-01, -1.4038e-01,  1.7297e-01,\n",
       "                       5.3760e-01, -1.8848e-01, -7.5244e-01, -2.4207e-01,  1.1115e-01,\n",
       "                      -3.4363e-02,  1.8469e-01,  4.0356e-01, -9.6436e-01,  5.1453e-02,\n",
       "                      -3.0835e-01,  8.6136e-03,  4.4482e-01, -5.2344e-01, -8.0273e-01,\n",
       "                      -2.3071e-01,  1.5125e-01,  3.0090e-02, -3.8452e-02,  6.4087e-02,\n",
       "                      -9.6338e-01,  2.9346e-01, -4.7144e-01,  2.1008e-01, -6.4209e-01,\n",
       "                      -4.0503e-01, -3.0762e-01,  1.3879e-01,  1.5088e-01,  2.6392e-01,\n",
       "                       4.7559e-01, -3.0688e-01,  2.7563e-01, -4.6777e-01, -6.7725e-01,\n",
       "                      -2.5098e+00,  3.5327e-01, -1.1846e+00, -9.4141e-01, -9.7510e-01,\n",
       "                      -2.3457e+00,  3.2886e-01, -1.2822e+00, -1.6445e+00,  4.0742e+00,\n",
       "                      -2.9766e+00,  2.8516e-01, -1.0518e+00, -8.2910e-01, -9.8242e-01,\n",
       "                       1.7490e+00,  1.7578e+00, -1.4619e+00,  6.0840e-01,  4.9609e-01,\n",
       "                      -1.9111e+00, -1.9658e+00,  1.4600e+00,  8.6768e-01,  8.5693e-02,\n",
       "                       5.9131e-01, -1.6631e+00, -1.6675e-01, -1.7041e+00,  2.5156e+00,\n",
       "                      -1.4473e+00, -2.5527e+00, -1.2646e+00,  8.0225e-01, -6.2891e-01,\n",
       "                      -6.9141e-01,  2.5723e+00, -3.9766e+00,  4.6240e-01, -1.7017e-01,\n",
       "                       5.2490e-01,  1.6152e+00, -2.8223e+00,  1.5002e-01, -7.9980e-01,\n",
       "                      -1.1777e+00,  7.1582e-01, -1.1768e+00,  5.7178e-01,  1.4819e-01,\n",
       "                       1.6309e+00, -8.3398e-01, -1.1826e-02, -1.7969e+00, -5.7666e-01,\n",
       "                      -1.2490e+00,  5.0879e-01,  1.2978e-02,  7.9688e-01,  1.4717e+00,\n",
       "                      -2.5625e+00,  9.5361e-01,  8.3594e-01,  1.1615e-01,  6.8262e-01,\n",
       "                      -2.9565e-01, -2.2715e+00, -7.4170e-01, -3.0249e-01, -2.1387e+00,\n",
       "                       2.1270e+00, -1.1416e+00, -6.9092e-01, -3.3643e-01, -6.5234e-01,\n",
       "                       2.6291e-02, -1.3506e+00, -6.1719e-01,  1.9226e-01,  5.0146e-01,\n",
       "                       3.5278e-01,  2.6543e+00, -1.2695e+00, -2.0050e-02, -4.9561e-01,\n",
       "                      -2.7324e+00, -9.9170e-01, -4.3066e-01, -1.4863e+00,  3.5614e-02,\n",
       "                      -1.2969e+00, -5.4150e-01,  2.3926e-01, -6.1670e-01,  3.2598e+00,\n",
       "                       1.9043e+00,  8.4863e-01,  7.9785e-01, -1.5247e-01,  2.4988e-01,\n",
       "                      -7.2412e-01,  3.8177e-02, -8.9551e-01, -1.8799e+00,  2.2241e-01,\n",
       "                      -7.3096e-01, -2.3848e+00,  9.6875e-01,  2.7051e-01,  1.7979e+00,\n",
       "                      -1.7939e+00,  5.6299e-01,  1.0381e+00, -6.3135e-01, -2.2090e+00,\n",
       "                      -2.3209e-02,  2.9414e+00,  3.7915e-01, -1.0039e+00, -2.7500e+00,\n",
       "                       2.1423e-01,  2.5605e+00,  5.8789e-01,  7.6318e-01,  5.9766e-01,\n",
       "                      -4.5215e-01, -2.3544e-02, -4.5280e-03, -2.1973e-03, -4.5242e-03,\n",
       "                      -4.0161e-02, -2.5005e-03,  1.2222e-02,  2.7828e-03,  7.1297e-03,\n",
       "                      -1.1955e-02,  9.3460e-03, -2.2369e-02,  2.1493e-04,  2.7252e-02,\n",
       "                       1.2833e-02, -1.7807e-02, -3.5278e-02,  1.7990e-02, -6.1646e-03,\n",
       "                       8.3637e-04,  1.0239e-02, -2.4204e-03, -1.1780e-01, -6.3705e-03,\n",
       "                       1.7776e-02,  5.3177e-03, -4.1107e-02, -4.2725e-02,  2.4487e-01,\n",
       "                       6.3934e-03,  1.7365e-02,  1.0114e-01, -1.2665e-02, -2.2232e-02,\n",
       "                       1.9913e-02,  1.0956e-02,  6.8092e-03, -8.4656e-02, -4.5090e-03,\n",
       "                       8.5526e-03,  2.7679e-02, -9.6893e-03, -5.4817e-03, -2.6581e-02,\n",
       "                      -2.3098e-03, -1.3771e-02,  2.0065e-02,  2.1362e-03,  7.8354e-03,\n",
       "                      -1.7914e-02,  6.2637e-03, -1.4429e-01, -3.0785e-03, -9.9335e-03,\n",
       "                       6.0310e-03, -2.2545e-03,  1.4238e-03,  7.0038e-03, -1.2207e-02,\n",
       "                      -9.5901e-03, -2.8366e-02, -1.1322e-01, -5.9448e-02, -1.3008e-02,\n",
       "                       1.5438e+01,  7.3438e-01, -8.4531e+00, -2.4328e+01, -1.6031e+01,\n",
       "                      -5.5195e+00, -5.8789e+00, -1.1930e+01, -1.1148e+01, -6.4609e+00,\n",
       "                       1.1773e+01,  1.7328e+01, -1.4641e+01, -2.0078e+01,  3.8320e+00,\n",
       "                       4.2109e+00,  1.5570e+01,  1.7920e+00,  7.7109e+00, -1.7719e+01,\n",
       "                       4.3555e+00, -5.3203e+00,  1.0023e+01, -3.8223e+00,  1.4000e+01,\n",
       "                       3.8418e+00, -6.6553e-01,  1.0516e+01, -1.3289e+01, -1.2242e+01,\n",
       "                      -8.4688e+00,  1.3016e+01,  2.1375e+01, -1.1164e+01, -8.8828e+00,\n",
       "                      -9.2734e+00,  5.7109e+00,  7.9141e+00, -1.5148e+01, -3.0688e-01,\n",
       "                       1.2938e+01,  4.1953e+00, -1.0703e+01,  2.8262e+00, -3.2637e+00,\n",
       "                      -5.6875e+00,  7.2812e+00,  7.6094e+00,  4.1445e+00, -8.9141e+00,\n",
       "                       3.5781e+00,  1.0453e+01, -8.6641e+00,  4.7812e+00, -5.3984e+00,\n",
       "                       1.2938e+01, -1.5891e+01,  8.0234e+00, -1.1578e+01,  2.4941e+00,\n",
       "                       9.3672e+00, -1.3805e+01,  1.1344e+01,  2.5000e+00,  2.4963e-02,\n",
       "                       1.9007e-03,  4.5197e-02,  3.1097e-02, -3.4218e-03, -1.3016e-02,\n",
       "                      -3.1677e-02, -2.0050e-02, -4.0619e-02, -2.2559e-01,  4.7531e-03,\n",
       "                       2.2263e-02,  1.0876e-01, -1.4854e-02,  5.3253e-02, -7.0267e-03,\n",
       "                       2.1545e-02, -4.3640e-02, -2.1225e-02,  2.8735e-01, -3.4302e-02,\n",
       "                      -1.2659e-01,  1.8372e-02,  3.5065e-02, -5.9265e-02, -2.0615e-02,\n",
       "                       1.5427e-02, -1.8585e-02,  5.2719e-03,  2.1835e-02,  1.7334e-02,\n",
       "                      -6.7566e-02, -2.1343e-03,  1.0162e-01,  9.1248e-03, -2.9449e-02,\n",
       "                       3.2318e-02, -2.3327e-03, -9.5444e-03,  1.4282e-02, -1.1612e-02,\n",
       "                      -1.7410e-02, -1.0663e-01,  1.5669e-03, -7.0129e-02, -3.2043e-02,\n",
       "                      -5.1537e-03,  1.0582e-02, -7.5035e-03, -2.2491e-02,  1.3876e-03,\n",
       "                      -1.4624e-01, -2.2526e-03,  1.6357e-02,  1.6980e-01,  1.1436e-02,\n",
       "                       4.1168e-02,  4.2297e-02, -4.9500e-02,  8.1787e-02,  7.2823e-03,\n",
       "                       1.5774e-03, -1.7395e-02,  3.2318e-02])),\n",
       "             ('transformer.h.0.self_attention.dense.weight',\n",
       "              tensor([[ 0.0122,  0.0003, -0.0229,  ..., -0.0079,  0.0089,  0.0069],\n",
       "                      [-0.0043, -0.0072, -0.0034,  ..., -0.0037, -0.0120, -0.0027],\n",
       "                      [ 0.0161,  0.0006,  0.0227,  ...,  0.0101, -0.0011, -0.0051],\n",
       "                      ...,\n",
       "                      [ 0.0042, -0.0124,  0.0009,  ...,  0.0039,  0.0158,  0.0029],\n",
       "                      [ 0.0029,  0.0047, -0.0086,  ..., -0.0017,  0.0059, -0.0127],\n",
       "                      [ 0.0228, -0.0119, -0.0216,  ..., -0.0006,  0.0134, -0.0172]])),\n",
       "             ('transformer.h.0.self_attention.dense.bias',\n",
       "              tensor([-0.0359, -0.0474, -0.0291,  ...,  0.3923, -0.0033,  0.0163])),\n",
       "             ('transformer.h.0.post_attention_layernorm.weight',\n",
       "              tensor([0.7017, 0.9473, 0.6328,  ..., 2.8965, 1.0088, 0.4961])),\n",
       "             ('transformer.h.0.post_attention_layernorm.bias',\n",
       "              tensor([-0.1055, -0.2959, -0.1288,  ..., -0.1609, -0.0515, -0.0800])),\n",
       "             ('transformer.h.0.mlp.dense_h_to_4h.weight',\n",
       "              tensor([[-0.0256,  0.0300,  0.0013,  ..., -0.0059,  0.0402,  0.0142],\n",
       "                      [-0.0190,  0.0088, -0.0148,  ...,  0.0086, -0.0234,  0.0168],\n",
       "                      [ 0.0020,  0.0234,  0.0399,  ...,  0.0080, -0.0114,  0.0287],\n",
       "                      ...,\n",
       "                      [-0.0223,  0.0103,  0.0089,  ...,  0.0170, -0.0250, -0.0323],\n",
       "                      [-0.0069,  0.0182, -0.0007,  ..., -0.0017, -0.0090,  0.0011],\n",
       "                      [ 0.0386,  0.0294,  0.0028,  ...,  0.0006, -0.0181,  0.0231]])),\n",
       "             ('transformer.h.0.mlp.dense_h_to_4h.bias',\n",
       "              tensor([-0.3311, -0.1938, -0.2085,  ..., -0.7446, -0.5107, -0.1307])),\n",
       "             ('transformer.h.0.mlp.dense_4h_to_h.weight',\n",
       "              tensor([[-0.0342,  0.0131,  0.0153,  ...,  0.0125, -0.0184,  0.0070],\n",
       "                      [ 0.0061,  0.0025, -0.0332,  ...,  0.0086,  0.0118,  0.0229],\n",
       "                      [ 0.0003,  0.0203, -0.0492,  ..., -0.0115, -0.0141, -0.0127],\n",
       "                      ...,\n",
       "                      [ 0.0019, -0.0054, -0.0191,  ...,  0.0079,  0.0227,  0.0050],\n",
       "                      [ 0.0199,  0.0103,  0.0084,  ..., -0.0113,  0.0055, -0.0006],\n",
       "                      [-0.0225,  0.0035, -0.0320,  ..., -0.0294,  0.0102,  0.0048]])),\n",
       "             ('transformer.h.0.mlp.dense_4h_to_h.bias',\n",
       "              tensor([ 0.1006, -0.0156,  0.0919,  ..., -0.0218, -0.1118, -0.0173])),\n",
       "             ('transformer.h.1.input_layernorm.weight',\n",
       "              tensor([0.8623, 1.0996, 0.8335,  ..., 2.1836, 1.2041, 0.7212])),\n",
       "             ('transformer.h.1.input_layernorm.bias',\n",
       "              tensor([-0.0718, -0.1479,  0.0300,  ...,  0.0742, -0.1066, -0.0033])),\n",
       "             ('transformer.h.1.self_attention.query_key_value.weight',\n",
       "              tensor([[ 0.0181, -0.0110,  0.0119,  ...,  0.0133,  0.0023, -0.0104],\n",
       "                      [ 0.0224, -0.0143, -0.0152,  ...,  0.0638,  0.0282,  0.0045],\n",
       "                      [-0.0156,  0.0095, -0.0161,  ...,  0.0654, -0.0231,  0.0338],\n",
       "                      ...,\n",
       "                      [ 0.0131, -0.0023, -0.0027,  ..., -0.0001,  0.0134,  0.0021],\n",
       "                      [-0.0004,  0.0110,  0.0050,  ..., -0.0130,  0.0009, -0.0030],\n",
       "                      [-0.0097, -0.0115,  0.0055,  ...,  0.0065, -0.0291,  0.0017]])),\n",
       "             ('transformer.h.1.self_attention.query_key_value.bias',\n",
       "              tensor([-1.6248e-01, -5.6305e-02, -1.2097e-01, -6.9641e-02,  1.8579e-01,\n",
       "                      -4.2458e-03,  5.0278e-03, -1.0028e-01,  9.2590e-02, -8.9661e-02,\n",
       "                      -3.5980e-02, -1.7432e-01, -9.6970e-03, -5.6543e-01, -1.3354e-01,\n",
       "                       9.2407e-02, -1.2100e-02,  2.4292e-01,  1.6736e-01,  2.6810e-02,\n",
       "                       4.0405e-01,  2.2259e-03, -1.2642e-02,  1.0791e-01, -1.3550e-01,\n",
       "                      -9.2957e-02, -1.2170e-01, -4.4128e-02, -7.5317e-02, -2.0218e-02,\n",
       "                      -1.0590e-01, -1.7139e-01, -1.7712e-01,  7.1228e-02, -2.4561e-01,\n",
       "                      -2.1230e+00, -1.4270e-01,  6.5491e-02,  1.2976e-01,  1.0278e-01,\n",
       "                      -5.9967e-02, -9.7900e-02,  4.0466e-02,  6.0547e-02, -7.3853e-02,\n",
       "                      -1.5087e-03,  2.0782e-02, -2.4750e-02, -4.1260e-02, -7.9407e-02,\n",
       "                      -2.4673e-02, -5.7953e-02, -4.5349e-02,  1.3171e-01, -9.5764e-02,\n",
       "                       1.0303e-01,  1.6125e-01, -1.0779e-01,  2.8784e-01,  1.9501e-02,\n",
       "                      -6.1676e-02,  9.4360e-02,  3.3051e-02, -1.3330e-01,  1.4941e-01,\n",
       "                       9.5947e-02,  3.1885e-01, -1.9928e-02, -2.1875e-01,  2.4506e-02,\n",
       "                      -4.0161e-02, -1.4197e-01,  2.4951e-01,  1.7456e-02,  8.4457e-03,\n",
       "                       1.4526e-01, -2.7515e-01, -1.8054e-01, -1.0315e-01, -4.9438e-02,\n",
       "                      -3.4027e-02, -6.0913e-02,  2.5977e+00,  4.2114e-02,  5.2032e-02,\n",
       "                       2.4744e-01,  1.9135e-02, -2.6270e-01, -2.0093e-01,  5.6549e-02,\n",
       "                       2.0605e-01, -4.3488e-02,  9.8694e-02,  4.9866e-02, -1.5735e-01,\n",
       "                       4.3640e-03, -2.2302e-01, -1.6321e-01,  1.5137e-01,  4.7028e-02,\n",
       "                      -1.5762e-02,  2.0813e-01, -7.6904e-02,  6.2073e-02,  9.4238e-02,\n",
       "                      -2.8931e-01, -7.8308e-02,  7.9651e-03, -1.4136e-01, -6.6345e-02,\n",
       "                       1.1102e-01, -6.9580e-01, -5.7434e-02,  2.4255e-01, -1.1896e-01,\n",
       "                      -9.9426e-02, -3.5492e-02, -6.3232e-02, -1.6675e-01,  4.4464e-02,\n",
       "                       1.3818e-01,  1.4380e-01, -2.3193e-01,  9.4482e-02,  4.5227e-02,\n",
       "                      -1.3782e-01,  8.1539e-04, -4.6356e-02,  5.9082e-02, -8.6823e-03,\n",
       "                      -9.1858e-03,  3.6659e-03,  5.6305e-02,  4.1595e-02, -4.3549e-02,\n",
       "                      -3.1189e-02, -2.6871e-02, -1.3359e-02, -1.4709e-02,  8.6441e-03,\n",
       "                       2.0004e-02,  3.2928e-02, -2.1076e-03, -2.3849e-02, -3.8403e-01,\n",
       "                       7.5745e-02,  1.2457e-04,  3.7708e-03,  3.2654e-02,  9.1553e-02,\n",
       "                      -7.7209e-03,  4.8447e-03, -1.9684e-02, -1.7029e-02, -1.0078e-02,\n",
       "                      -3.1799e-02,  4.8790e-03, -3.0151e-02, -5.2521e-02, -3.4149e-02,\n",
       "                       1.4450e-02, -4.3579e-02,  5.2521e-02,  5.3444e-03,  8.0322e-02,\n",
       "                       1.2146e-01, -4.7119e-02, -5.4779e-03, -3.8891e-03,  1.0864e-02,\n",
       "                      -1.1627e-02,  6.9214e-02, -1.7910e+00,  9.0454e-02, -3.7811e-02,\n",
       "                       2.0523e-03,  1.2596e-02, -2.3224e-02,  5.5084e-03, -3.3887e-01,\n",
       "                       9.0576e-02, -1.1757e-02, -1.7044e-02, -2.6428e-02,  1.2505e-02,\n",
       "                      -2.4536e-02, -3.7899e-03, -4.2023e-02,  1.7252e-03, -4.5361e-01,\n",
       "                       1.4087e-01,  1.9653e-02,  3.3276e-01,  1.7908e-01,  3.9734e-02,\n",
       "                       5.8496e-01, -7.6465e-01, -1.3684e-01, -5.8984e-01, -1.3550e-01,\n",
       "                       7.1350e-02,  4.2627e-01,  3.9233e-01, -1.6543e+00, -1.9016e-03,\n",
       "                       1.5417e-01, -1.3367e-01, -2.3315e-01,  1.5225e+00,  5.1807e-01,\n",
       "                       1.7590e-01, -1.8030e-01, -9.3555e-04, -2.3669e-01, -3.6011e-01,\n",
       "                       6.4270e-02, -2.7199e-03, -6.8665e-02,  1.2012e-01,  6.5186e-01,\n",
       "                      -1.1559e-02,  3.3765e-01,  4.3262e-01, -3.7354e-02,  4.4482e-01,\n",
       "                      -4.4800e-01,  5.4590e-01,  1.0236e-01, -2.7075e-01,  2.1521e-01,\n",
       "                       1.8640e-01,  2.0485e-03, -1.7078e-01, -2.4756e-01,  3.8696e-01,\n",
       "                      -3.3789e-01,  2.4255e-01,  6.6699e-01,  2.4500e-01,  5.4590e-01,\n",
       "                       1.1786e-01,  8.0078e-02,  3.6304e-01,  4.1333e-01, -2.0398e-01,\n",
       "                       1.0996e+00, -2.3096e-01, -4.5825e-01, -3.6035e-01, -2.0294e-02,\n",
       "                      -4.5312e-01,  3.9697e-01, -2.5757e-01, -1.5991e-01, -9.4116e-02,\n",
       "                       3.1055e-01,  2.3962e-01,  3.4790e-01,  2.3682e-01, -1.7090e-02,\n",
       "                       7.3128e-03, -1.6431e-01, -1.6907e-01, -1.3696e-01,  9.0698e-02,\n",
       "                      -2.5415e-01, -4.3359e-01, -1.3164e+00, -1.2073e-01,  5.3564e-01,\n",
       "                      -3.5370e-02, -6.1127e-02, -7.3779e-01, -1.9934e-01,  6.8298e-02,\n",
       "                      -2.4536e-01,  6.7017e-02, -6.2744e-01,  1.3196e-01,  1.7468e-01,\n",
       "                      -3.7061e-01,  3.9990e-01,  4.2944e-01,  2.1387e-01, -1.0138e-01,\n",
       "                      -8.5266e-02, -9.3689e-02,  1.8079e-01,  2.0728e-01, -1.6084e+00,\n",
       "                       1.1798e-01, -5.9521e-01,  2.1313e-01, -2.7222e-01, -1.7041e-01,\n",
       "                      -1.0938e-01, -7.3669e-02, -8.4424e-01,  1.5747e-01,  1.6211e-01,\n",
       "                       4.9048e-01, -2.7161e-02,  4.1229e-02,  3.0469e-01, -2.1887e-01,\n",
       "                      -3.1967e-03,  1.0925e-01, -3.5522e-01, -2.3230e-01, -3.5083e-01,\n",
       "                      -8.0643e-03, -2.6099e-01, -7.1411e-02,  2.2925e-01, -7.1838e-02,\n",
       "                       9.5520e-02, -2.9614e-01, -1.9885e-01,  2.1790e-01,  1.3733e-01,\n",
       "                      -7.8678e-04,  8.6823e-03,  4.0009e-02, -8.6899e-03,  1.4915e-02,\n",
       "                      -1.1993e-02,  4.2381e-03,  1.7502e-02, -2.9144e-03,  2.5528e-02,\n",
       "                       1.3664e-02, -2.4292e-02, -4.1275e-03,  5.9540e-02,  3.1158e-02,\n",
       "                       1.9897e-02, -2.9709e-02, -2.3865e-02,  1.6413e-03,  2.4231e-02,\n",
       "                       1.0170e-02, -2.5482e-02, -1.6815e-02,  4.1466e-03,  2.0340e-02,\n",
       "                      -5.0568e-02,  9.9182e-03,  3.1555e-02,  2.1362e-02, -1.6129e-02,\n",
       "                       2.1088e-02,  2.8137e-02, -2.5314e-02, -1.6418e-02,  1.2405e-02,\n",
       "                       2.1576e-02, -8.7967e-03, -1.4565e-02,  1.0208e-02, -7.7972e-03,\n",
       "                       2.1343e-03, -1.8600e-02, -4.0207e-03, -1.0902e-02, -4.7183e-04,\n",
       "                      -1.1795e-02, -2.3895e-02,  7.5912e-03, -2.3127e-04,  1.1688e-02,\n",
       "                       3.0289e-02,  4.5807e-02,  2.8839e-02, -4.9164e-02, -4.1389e-03,\n",
       "                       3.3936e-02, -1.3832e-02,  1.6876e-02,  3.6530e-02, -6.6833e-03,\n",
       "                      -1.5915e-02, -1.4153e-02, -3.4668e-02,  7.3357e-03, -2.9907e-01,\n",
       "                       3.7671e-01,  1.3599e-01, -3.9868e-01,  4.4403e-03,  9.6359e-03,\n",
       "                       1.4197e-01,  4.1357e-01, -7.1436e-01,  1.4641e-02,  2.8882e-01,\n",
       "                      -1.2494e-01, -2.3010e-02,  9.8486e-01, -1.2964e-01,  6.2103e-02,\n",
       "                       1.1975e-01, -2.9517e-01, -1.2207e-01, -3.7012e-01,  6.8298e-02,\n",
       "                       1.1375e-02,  4.5728e-01, -3.1982e-01,  2.4146e-01, -2.2552e-02,\n",
       "                      -6.6101e-02,  2.3392e-02, -1.6003e-01,  3.8159e-01,  4.3311e-01,\n",
       "                       3.6652e-02, -2.7856e-01, -2.3499e-01, -2.9297e-01,  2.2888e-01,\n",
       "                       3.0957e-01,  2.5903e-01,  5.9424e-01, -1.4307e-01, -1.5297e-02,\n",
       "                      -2.6514e-01, -4.3970e-01, -1.6638e-01,  2.2986e-01, -1.6052e-02,\n",
       "                      -1.2598e+00, -6.8750e-01, -1.8567e-01,  8.6243e-02,  2.5171e-01,\n",
       "                      -7.3303e-02,  1.1353e-01,  5.5939e-02,  1.6479e-01, -5.6738e-01,\n",
       "                      -7.0703e-01,  4.7266e-01,  9.0576e-02, -7.6416e-01,  4.6948e-01,\n",
       "                      -1.9897e-02, -6.1572e-01,  2.1936e-01, -1.9424e+00, -5.2002e-01,\n",
       "                       1.2012e+00,  7.8662e-01,  3.3966e-02, -1.0625e+00,  3.4180e-01,\n",
       "                       1.4502e+00,  4.5679e-01,  1.1826e+00, -6.9287e-01,  1.5342e+00,\n",
       "                      -9.9316e-01, -4.2871e-01,  5.8545e-01,  1.5381e+00,  2.7612e-01,\n",
       "                      -7.9883e-01, -2.1655e-01, -5.6055e-01, -1.1650e+00,  1.6992e+00,\n",
       "                      -1.1523e+00,  1.1904e+00, -7.5293e-01, -3.9355e-01, -7.4707e-01,\n",
       "                       8.7842e-01,  2.5708e-01,  1.4832e-01, -3.0640e-01, -7.8369e-01,\n",
       "                      -5.6055e-01,  4.7827e-01, -7.2998e-01, -2.1851e-01, -1.1810e-01,\n",
       "                      -4.6729e-01,  1.1475e+00,  5.8545e-01,  8.7354e-01,  8.6572e-01,\n",
       "                       1.5723e+00,  6.6943e-01, -1.9373e-01, -1.8848e-01, -2.8105e+00,\n",
       "                       5.8252e-01, -4.2114e-01,  4.5630e-01, -1.4062e+00, -3.0273e-01,\n",
       "                       1.1224e-01, -2.0728e-01, -2.7420e-02,  1.2402e+00, -2.4670e-01,\n",
       "                      -1.1709e+00,  1.4664e-02, -2.0859e+00,  3.0420e-01, -2.7051e-01,\n",
       "                      -3.0566e+00,  7.2852e-01,  5.8685e-02, -9.4528e-03,  3.2449e-04,\n",
       "                       8.2886e-02,  6.5308e-03,  1.6718e-03, -1.2543e-02, -2.3270e-02,\n",
       "                      -8.3740e-02,  1.1505e-01,  1.0242e-03, -2.3163e-02, -3.7231e-02,\n",
       "                       2.2079e-02,  2.0447e-02, -1.3611e-02,  1.5099e-02, -1.8906e-02,\n",
       "                       1.4664e-02, -1.8997e-02,  1.9974e-02, -3.0884e-02, -1.8326e-02,\n",
       "                       2.3544e-02, -2.6627e-02, -1.8481e-01,  2.7695e-02,  6.8321e-03,\n",
       "                       4.6577e-03,  1.7883e-02,  6.0158e-03, -1.6571e-02, -3.4180e-01,\n",
       "                      -8.9661e-02, -3.6469e-02, -5.1086e-02,  8.8196e-02,  9.1400e-03,\n",
       "                      -5.6610e-02, -6.1920e-02,  1.6388e-02, -1.7914e-02, -1.8585e-02,\n",
       "                      -1.7288e-02,  5.7922e-02,  1.1188e-01,  1.3626e-02, -4.8920e-02,\n",
       "                       1.5976e-02, -6.9504e-03, -7.7477e-03, -5.1331e-02,  2.8725e-03,\n",
       "                       3.8757e-02,  2.5375e-02, -1.0536e-02,  1.2238e-02, -3.9635e-03,\n",
       "                      -4.8714e-03,  2.5940e-02,  9.8419e-03,  2.0676e-02,  2.3300e-02,\n",
       "                      -1.0361e-02, -7.0264e-01, -8.7708e-02,  1.0345e-01, -2.2913e-01,\n",
       "                      -5.5469e-01,  4.3506e-01, -1.3513e-01, -5.0391e-01,  1.6495e-02,\n",
       "                       4.5605e-01, -2.8198e-01, -6.7017e-02, -4.9316e-02,  1.1151e-01,\n",
       "                       1.9238e-01,  9.0027e-02, -1.7456e-01,  3.1433e-02, -2.2034e-01,\n",
       "                      -1.4563e-01, -1.5637e-01, -2.1088e-02,  2.0416e-02,  2.4988e-01,\n",
       "                      -7.5928e-01, -2.8760e-01,  4.3701e-01,  4.9805e-01, -2.4487e-01,\n",
       "                       4.7485e-01,  4.8169e-01, -3.5132e-01,  2.6074e-01, -3.5620e-01,\n",
       "                       1.6724e-02,  3.9429e-01, -1.2656e+00,  8.2581e-02, -2.0032e-01,\n",
       "                       3.4302e-01, -6.6162e-01,  3.6438e-02, -3.6353e-01, -2.1069e-01,\n",
       "                       6.4453e-02, -1.1566e-01, -6.7291e-03, -8.6523e-01,  4.0234e-01,\n",
       "                       9.1324e-03, -3.7402e-01,  5.8301e-01, -6.4331e-02,  3.0713e-01,\n",
       "                       8.4167e-02, -4.0503e-01,  4.9561e-01, -1.3196e-01, -3.3496e-01,\n",
       "                      -5.2979e-01, -2.9565e-01, -5.0879e-01,  1.1566e-01, -3.4088e-02,\n",
       "                      -1.9414e+00, -5.7266e+00,  4.6997e-01,  1.6143e+00, -5.8711e+00,\n",
       "                       8.2227e-01,  2.9023e+00, -2.0664e+00,  1.3174e+00,  2.4160e+00,\n",
       "                      -3.6523e+00, -4.1562e+00,  3.2324e+00,  1.1055e+00, -1.7266e+00,\n",
       "                      -1.3965e+00,  4.6289e-01, -2.5171e-01, -5.6758e+00,  1.6260e+00,\n",
       "                       2.3206e-01, -1.3464e-01, -1.3740e+00, -1.5508e+00, -5.4980e-01,\n",
       "                      -4.1406e+00,  1.2002e+00,  1.0625e+00, -6.5723e-01,  3.4648e+00,\n",
       "                       1.7295e+00, -4.1914e+00, -1.8242e+00, -9.8682e-01,  1.5586e+00,\n",
       "                       3.1226e-01, -3.8891e-03,  2.0508e-02,  7.1289e-02, -1.8701e+00,\n",
       "                       6.2383e+00, -1.4014e+00,  3.0527e+00, -1.2129e+00, -3.4238e+00,\n",
       "                       2.7168e+00,  4.3042e-01,  5.0342e-01, -5.7709e-02, -2.9980e+00,\n",
       "                      -3.2773e+00,  3.2930e+00,  6.2744e-02, -4.4141e+00,  3.6562e+00,\n",
       "                      -2.7612e-01,  2.2168e+00,  1.2500e+00, -2.7988e+00,  2.2930e+00,\n",
       "                      -1.0410e+00, -2.6484e+00,  8.3936e-01, -5.1953e-01,  4.5853e-03,\n",
       "                       3.1113e-02, -2.9358e-02,  5.4230e-02, -2.3071e-02, -1.8173e-02,\n",
       "                      -7.6294e-02, -3.4515e-02, -4.8752e-03,  6.5536e-03,  3.1235e-02,\n",
       "                      -1.9470e-02,  4.4952e-02,  2.7863e-02, -6.1188e-03, -2.6321e-02,\n",
       "                      -1.9836e-02, -1.4791e-03,  4.1290e-02,  1.9714e-02, -1.5030e-03,\n",
       "                       1.5198e-02, -7.6172e-02, -1.6678e-02,  1.7014e-02,  1.6129e-02,\n",
       "                       2.1582e-01,  4.1809e-03, -3.8727e-02,  7.0679e-02, -2.3102e-02,\n",
       "                      -2.5320e-04, -6.0425e-03,  1.2128e-01,  2.3376e-02, -3.1909e-01,\n",
       "                       7.8321e-05,  5.8167e-02,  1.1948e-02,  1.4331e-01, -5.3711e-03,\n",
       "                       2.2046e-01,  1.4305e-02,  7.3425e-02, -3.0231e-03,  4.2999e-02,\n",
       "                       6.5979e-02, -2.2156e-02, -8.5754e-03,  2.4475e-02, -4.4067e-02,\n",
       "                      -3.6201e-03, -5.2002e-02,  7.1602e-03, -8.2169e-03, -5.3940e-03,\n",
       "                       1.6833e-01,  2.8152e-02, -1.1673e-02,  1.5540e-01, -5.1613e-03,\n",
       "                      -2.9373e-02,  6.0211e-02, -3.2227e-02])),\n",
       "             ('transformer.h.1.self_attention.dense.weight',\n",
       "              tensor([[-1.4091e-02, -1.1383e-02, -2.9793e-03,  ..., -9.9945e-03,\n",
       "                       -4.8943e-03,  2.1400e-03],\n",
       "                      [-1.6632e-03, -7.5607e-03, -3.5782e-03,  ...,  4.3335e-03,\n",
       "                       -2.2690e-02,  6.8359e-03],\n",
       "                      [ 1.9516e-02, -2.2568e-02,  5.7068e-03,  ..., -1.2222e-02,\n",
       "                       -5.8250e-03, -7.8583e-03],\n",
       "                      ...,\n",
       "                      [-3.0731e-02,  1.1482e-02, -9.8801e-03,  ..., -7.1678e-03,\n",
       "                        6.7444e-03,  1.3489e-02],\n",
       "                      [ 2.7428e-03, -1.0887e-02,  6.6338e-03,  ...,  2.5970e-02,\n",
       "                       -1.4801e-02,  9.2392e-03],\n",
       "                      [ 6.7532e-05,  1.5137e-02, -1.3161e-03,  ..., -9.4299e-03,\n",
       "                       -6.6719e-03,  9.6679e-05]])),\n",
       "             ('transformer.h.1.self_attention.dense.bias',\n",
       "              tensor([-0.0263, -0.0421,  0.0184,  ...,  0.0314,  0.0030,  0.0209])),\n",
       "             ('transformer.h.1.post_attention_layernorm.weight',\n",
       "              tensor([0.8101, 1.0615, 0.8179,  ..., 2.0391, 1.2588, 0.5947])),\n",
       "             ('transformer.h.1.post_attention_layernorm.bias',\n",
       "              tensor([-0.0660, -0.1742, -0.3064,  ..., -0.2515, -0.2991, -0.1224])),\n",
       "             ('transformer.h.1.mlp.dense_h_to_4h.weight',\n",
       "              tensor([[ 1.2787e-02,  8.2474e-03, -2.0599e-02,  ..., -2.7863e-02,\n",
       "                        4.8599e-03,  5.0240e-03],\n",
       "                      [-1.3161e-02, -3.1647e-02,  1.8234e-02,  ..., -1.8778e-03,\n",
       "                       -2.4475e-02,  9.9945e-03],\n",
       "                      [-1.5404e-02,  2.7039e-02, -3.4733e-03,  ...,  2.1545e-02,\n",
       "                        1.9714e-02,  6.9885e-03],\n",
       "                      ...,\n",
       "                      [ 2.7447e-03, -2.3727e-02,  1.0796e-02,  ...,  2.9251e-02,\n",
       "                       -2.0599e-02, -8.0526e-05],\n",
       "                      [-6.5956e-03,  9.5978e-03, -3.0914e-02,  ...,  1.4257e-03,\n",
       "                       -1.8234e-02, -1.2680e-02],\n",
       "                      [ 1.3878e-02, -9.9182e-04, -1.2993e-02,  ..., -1.0330e-02,\n",
       "                       -2.5978e-03,  3.2990e-02]])),\n",
       "             ('transformer.h.1.mlp.dense_h_to_4h.bias',\n",
       "              tensor([-0.1027, -0.1183, -0.5703,  ..., -0.2942, -0.4082, -0.2820])),\n",
       "             ('transformer.h.1.mlp.dense_4h_to_h.weight',\n",
       "              tensor([[ 2.3331e-02,  1.9588e-03,  1.2802e-02,  ...,  9.3536e-03,\n",
       "                       -1.9882e-02,  5.0240e-03],\n",
       "                      [ 6.7520e-03,  2.2202e-02, -1.8341e-02,  ...,  2.7275e-03,\n",
       "                       -2.2736e-02, -1.7044e-02],\n",
       "                      [ 2.3022e-03, -1.4374e-02, -3.5267e-03,  ...,  7.7095e-03,\n",
       "                       -2.6016e-02,  5.4598e-05],\n",
       "                      ...,\n",
       "                      [ 7.3547e-03, -7.9422e-03, -2.3972e-02,  ...,  2.5238e-02,\n",
       "                       -1.0254e-02,  2.3712e-02],\n",
       "                      [-2.7542e-03,  8.6975e-03,  8.9645e-03,  ..., -9.2087e-03,\n",
       "                       -1.1566e-02,  3.2783e-04],\n",
       "                      [ 7.9956e-03, -2.4139e-02,  8.3084e-03,  ..., -2.7969e-02,\n",
       "                       -6.9008e-03,  1.5793e-02]])),\n",
       "             ('transformer.h.1.mlp.dense_4h_to_h.bias',\n",
       "              tensor([ 0.0589, -0.0735,  0.0988,  ..., -0.1067, -0.1306, -0.0177])),\n",
       "             ('transformer.h.2.input_layernorm.weight',\n",
       "              tensor([0.9370, 1.1934, 0.8804,  ..., 2.0547, 1.3408, 0.6797])),\n",
       "             ('transformer.h.2.input_layernorm.bias',\n",
       "              tensor([-0.0654, -0.1307,  0.0129,  ...,  0.0402, -0.1169,  0.0181])),\n",
       "             ('transformer.h.2.self_attention.query_key_value.weight',\n",
       "              tensor([[ 0.0039, -0.0276, -0.0231,  ..., -0.0119, -0.0141,  0.0041],\n",
       "                      [ 0.0038,  0.0198, -0.0038,  ..., -0.0312,  0.0455, -0.0656],\n",
       "                      [ 0.0163, -0.0390,  0.0081,  ...,  0.0038,  0.0058,  0.0299],\n",
       "                      ...,\n",
       "                      [-0.0115,  0.0291, -0.0085,  ..., -0.0299,  0.0376,  0.0208],\n",
       "                      [ 0.0295, -0.0152, -0.0667,  ..., -0.0113, -0.0291,  0.0207],\n",
       "                      [-0.0467,  0.0421,  0.0127,  ..., -0.0037,  0.0175,  0.0623]])),\n",
       "             ('transformer.h.2.self_attention.query_key_value.bias',\n",
       "              tensor([ 7.2876e-02, -5.8319e-02, -3.8757e-02,  6.6406e-02,  1.7624e-02,\n",
       "                       3.8940e-02, -6.7902e-03,  2.6306e-02,  2.7657e-03, -1.2238e-02,\n",
       "                       8.1665e-02,  1.3214e-02, -4.1046e-02, -8.3008e-03, -3.7136e-03,\n",
       "                      -1.3809e-02,  3.4393e-02, -1.7395e-03, -3.5675e-02, -1.8501e-03,\n",
       "                      -6.0394e-02, -7.6782e-02,  2.8152e-02, -2.9964e-03, -2.5005e-03,\n",
       "                      -2.1378e-02,  1.2672e-02, -1.5793e-02,  7.8964e-03, -5.0323e-02,\n",
       "                       1.3138e-02, -2.0477e-02,  5.2032e-03, -3.5950e-02, -3.5553e-02,\n",
       "                      -6.1607e-03,  7.3425e-02,  4.1008e-03,  1.0483e-02, -1.8311e-02,\n",
       "                       2.7390e-03, -1.2939e-02,  5.5359e-02, -6.8298e-02, -1.5507e-03,\n",
       "                      -2.6493e-03, -9.3918e-03,  7.9712e-02,  5.6885e-02, -3.1647e-02,\n",
       "                       4.0833e-02,  3.3142e-02, -6.1369e-04,  1.7456e-02, -1.8677e-02,\n",
       "                       1.8768e-02,  5.1300e-02, -1.9745e-02, -3.0014e-02, -5.8937e-03,\n",
       "                      -2.6031e-02,  3.0853e-02,  2.5768e-03, -3.0197e-02,  2.3523e-01,\n",
       "                      -8.6609e-02, -6.5651e-03, -6.2500e-01,  2.4023e-01, -3.6792e-01,\n",
       "                      -2.1713e-02, -1.1244e-03, -1.4229e+00,  2.6685e-01,  9.4238e-02,\n",
       "                      -2.8003e-01,  1.3904e-01, -9.0479e-01,  1.8420e-01, -3.4790e-01,\n",
       "                       2.4988e-01, -4.8682e-01,  1.4844e-01, -5.7129e-01,  2.9761e-01,\n",
       "                      -1.6821e-01,  7.3975e-02, -5.4541e-01, -5.2246e-01, -4.4360e-01,\n",
       "                       1.1249e-01, -2.4731e-01, -5.0446e-02, -3.3521e-01,  1.7529e-01,\n",
       "                       7.2949e-01,  3.4351e-01,  2.9688e-01, -2.1045e-01,  2.3392e-02,\n",
       "                      -2.3376e-01,  9.8022e-02, -1.3330e-01, -7.7515e-03,  1.9336e-01,\n",
       "                       4.4360e-01, -6.0938e-01, -3.3960e-01, -2.6050e-01,  2.1863e-01,\n",
       "                       5.3857e-01, -3.5986e-01, -3.3057e-01,  4.6448e-02, -4.5361e-01,\n",
       "                      -2.4033e-03, -4.4043e-01, -2.8296e-01,  1.1591e-01,  2.8320e-01,\n",
       "                       3.9404e-01,  1.7520e+00,  4.5190e-01,  2.0178e-01,  4.8975e-01,\n",
       "                       2.6779e-02,  1.7517e-01,  1.5942e-01, -8.1934e-01,  2.1408e-02,\n",
       "                      -3.3741e-03,  1.7502e-02,  2.4986e-03,  2.3605e-02,  2.0432e-02,\n",
       "                      -9.2850e-03, -6.7673e-03,  3.7811e-02,  1.3599e-01,  3.1769e-02,\n",
       "                      -1.6235e-02, -8.1635e-03, -3.4393e-02, -6.7024e-03,  3.0469e-01,\n",
       "                       1.8784e-02, -2.2446e-02,  1.5961e-02, -1.2367e-02, -1.9470e-02,\n",
       "                      -3.3417e-02, -1.6571e-02,  1.5063e-01, -3.5248e-02, -5.6305e-03,\n",
       "                       4.2267e-03,  1.7548e-02,  7.2432e-04, -1.6953e-02, -2.1500e-02,\n",
       "                      -1.0956e-02, -4.8904e-03, -1.2886e-02,  3.7720e-02, -5.1709e-01,\n",
       "                      -1.2794e-02, -1.3375e-04,  2.7573e-02,  1.5717e-02, -1.1587e-03,\n",
       "                      -9.8114e-03, -3.4973e-02,  4.2319e-04, -1.3313e-03,  2.0096e-02,\n",
       "                      -2.6810e-02,  7.3776e-03, -4.0009e-02, -1.4984e-02,  1.6632e-02,\n",
       "                       2.8885e-02,  4.3701e-02,  5.0232e-02, -4.0070e-02, -3.8490e-03,\n",
       "                      -3.4882e-02, -6.9031e-02,  1.1559e-02, -4.6120e-03, -4.0245e-03,\n",
       "                       1.3054e-02, -2.6749e-02,  6.8512e-03,  1.9055e-01,  1.0809e-01,\n",
       "                       2.5928e-01,  2.7563e-01,  1.9519e-01,  3.7427e-01, -3.7231e-01,\n",
       "                      -2.7832e-01, -1.1243e-01, -3.6011e-01, -8.8440e-02,  1.3466e-03,\n",
       "                      -3.4637e-02,  2.7161e-02, -4.1699e-01, -2.8735e-01,  4.1406e-01,\n",
       "                      -1.0809e-01, -3.0945e-02, -2.8003e-01, -7.2021e-02,  4.7656e-01,\n",
       "                      -1.0797e-01,  7.9150e-01,  1.4441e-01, -1.2856e-02, -8.3679e-02,\n",
       "                       9.3445e-02, -8.7402e-01,  1.7090e-01,  6.0303e-01,  1.5234e-01,\n",
       "                      -5.8105e-01, -1.0254e-01, -3.0396e-01,  5.4980e-01,  4.3182e-02,\n",
       "                       3.5156e-02, -7.4730e-03, -8.1116e-02, -7.0992e-03, -1.5430e-01,\n",
       "                      -1.1133e-01,  6.5125e-02,  1.4282e-01, -1.6650e-01,  3.5693e-01,\n",
       "                       3.0566e-01, -2.0264e-01, -3.3228e-01, -5.2637e-01, -2.5977e-01,\n",
       "                       2.6099e-01, -6.3477e-02, -9.9854e-02, -1.0913e-01, -1.4062e-01,\n",
       "                      -1.1040e-02, -1.9421e-01, -1.4246e-01, -2.4841e-02,  2.0105e-01,\n",
       "                       4.9042e-02, -1.3367e-02, -3.7506e-02, -1.2006e-01,  1.1986e-02,\n",
       "                      -2.1133e-02,  1.1246e-02, -1.0864e-02,  3.2158e-03,  1.1024e-02,\n",
       "                       4.5090e-03, -7.9285e-02,  1.9485e-02, -3.5797e-02,  2.3529e-02,\n",
       "                       3.3783e-02, -2.0035e-02,  1.3725e-02,  3.7079e-02,  2.2583e-02,\n",
       "                       1.1497e-02,  1.9379e-02,  7.2205e-02, -6.5117e-03,  2.6016e-02,\n",
       "                       1.3647e-01, -4.0894e-02,  4.8523e-02,  3.7354e-02, -4.2725e-02,\n",
       "                      -1.7639e-02, -2.0233e-02, -3.0502e-02,  2.3460e-03, -3.4668e-02,\n",
       "                      -2.4811e-02, -4.5624e-02, -3.9893e-01,  3.3045e-04,  8.3740e-02,\n",
       "                       2.2507e-02, -3.0624e-02, -1.7212e-02,  1.3229e-02,  1.8631e-02,\n",
       "                       3.7842e-03, -1.8005e-02,  2.9739e-02, -2.7634e-02,  2.8992e-02,\n",
       "                       3.1860e-02,  4.3678e-04,  3.3661e-02,  2.7298e-02, -3.7872e-02,\n",
       "                      -3.3203e-02, -7.7637e-02,  1.1368e-02, -1.1406e-02, -1.4336e-02,\n",
       "                      -5.1910e-02, -1.9318e-02, -2.5803e-02, -2.4445e-02,  1.6678e-02,\n",
       "                      -1.3940e-01, -1.0576e+00,  3.4253e-01, -4.7046e-01,  2.1790e-01,\n",
       "                      -9.2468e-03, -4.0161e-01, -1.6150e-01, -2.8574e+00, -4.1772e-01,\n",
       "                       5.2704e-02,  2.8174e-01,  8.7891e-03,  1.3123e-01,  2.2578e+00,\n",
       "                      -1.6504e-01,  3.0518e-01,  1.6980e-01, -1.1127e-01, -9.6863e-02,\n",
       "                       1.8274e-01, -7.1899e-02, -1.8298e-01,  1.1389e-01,  4.8615e-02,\n",
       "                      -1.5503e-01, -3.4741e-01,  3.6465e+00, -3.7891e+00,  1.4755e-02,\n",
       "                       2.1692e-01,  9.1934e-03, -2.5342e-01,  1.8945e-01,  1.8555e-01,\n",
       "                      -3.2007e-01, -2.8271e-01, -2.6855e-01, -2.3462e-01,  7.0215e-01,\n",
       "                       7.1106e-02, -5.1221e-01,  8.1970e-02,  1.1011e-01,  3.5571e-01,\n",
       "                      -1.8274e-01, -3.0396e-01, -1.0968e-01, -1.4905e-01, -1.6760e-01,\n",
       "                      -2.1997e-01,  8.4991e-03,  1.8713e-01, -1.0063e-02, -6.5613e-02,\n",
       "                       1.3806e-01, -6.7871e-02, -5.2246e-02, -2.8183e-02,  4.6240e-01,\n",
       "                      -8.6975e-03,  4.9324e-03, -2.1130e-01, -2.1042e-02, -1.7334e-01,\n",
       "                       1.6632e-02, -3.0493e-01,  8.9905e-02,  1.8253e-03, -1.0437e-01,\n",
       "                      -4.6826e-01,  1.4954e-01,  4.0771e-01,  7.0508e-01, -8.2153e-02,\n",
       "                      -1.6846e-01,  4.2017e-01,  4.0771e-01,  6.8506e-01, -8.6719e-01,\n",
       "                       1.5808e-01,  2.7954e-01,  4.1718e-02,  3.6572e-01,  1.0742e-01,\n",
       "                      -1.5808e-01,  5.2295e-01, -3.0786e-01,  1.4746e-01, -1.3464e-01,\n",
       "                       4.9805e-02, -2.4597e-01, -7.5244e-01, -6.9336e-01,  3.1372e-01,\n",
       "                      -1.8970e-01, -2.1301e-01,  4.4482e-01,  3.1982e-01,  5.2490e-02,\n",
       "                      -2.1521e-01,  4.7455e-02, -9.4873e-01,  5.0568e-02, -9.1113e-01,\n",
       "                       1.1768e-03,  1.2627e-02,  1.5588e-01,  7.2813e-04,  5.1025e-01,\n",
       "                       1.9177e-01,  1.7899e-02,  1.7090e-01,  6.0425e-02,  2.6685e-01,\n",
       "                       1.9092e-01, -1.2805e-01, -2.9468e-01,  7.5623e-02, -2.8296e-01,\n",
       "                       7.2266e-02,  1.5540e-01,  4.3848e-01, -2.4648e+00, -1.1591e-01,\n",
       "                      -4.1431e-01,  1.8567e-01,  2.8076e-01, -1.2054e-02,  2.3727e-02,\n",
       "                      -1.8539e-02, -2.6031e-02, -3.4218e-03,  3.6240e-03,  1.9653e-02,\n",
       "                       3.0838e-02, -9.0637e-03, -1.7042e-03,  6.7215e-03,  1.6632e-02,\n",
       "                      -4.8637e-03, -1.3695e-02,  1.2100e-02,  3.0075e-02,  1.9058e-02,\n",
       "                       8.6594e-03,  6.2752e-03, -1.4397e-02,  5.0316e-03,  1.4687e-02,\n",
       "                      -5.8651e-04,  2.7267e-02, -7.8430e-03,  1.0223e-02,  1.6312e-02,\n",
       "                       3.2593e-02,  3.0594e-02, -2.0401e-02,  2.8091e-02, -2.0660e-02,\n",
       "                      -1.2405e-02, -5.3329e-03,  4.1509e-04, -6.9542e-03, -2.8343e-03,\n",
       "                      -1.4053e-02, -3.0861e-03, -7.9956e-03,  1.1871e-02,  4.4373e-02,\n",
       "                      -5.3345e-02, -1.9485e-02,  1.0727e-02, -2.7752e-03,  1.6464e-02,\n",
       "                      -5.3139e-03, -6.0463e-03,  1.5503e-02,  1.4046e-02, -3.7384e-03,\n",
       "                       7.4387e-03,  1.8982e-02, -1.1497e-02, -3.6526e-03, -1.8005e-02,\n",
       "                       1.0590e-02,  2.2240e-03, -2.8343e-03, -9.6054e-03, -8.9798e-03,\n",
       "                      -4.0314e-02, -2.1877e-03, -3.7427e-01,  1.4832e-02,  1.7493e-01,\n",
       "                       4.6777e-01, -1.7242e-02,  6.8420e-02, -5.7178e-01, -1.1191e+00,\n",
       "                       5.0000e-01, -1.5830e+00,  4.3579e-01,  4.9170e-01, -6.1066e-02,\n",
       "                       1.7212e-01, -1.4014e-01,  1.6602e-01, -2.2266e-01,  1.2268e-01,\n",
       "                      -2.4377e-01,  5.8740e-01, -2.5317e-01, -1.5991e-01,  3.6041e-02,\n",
       "                      -5.4993e-02,  8.9966e-02, -4.4580e-01,  1.1444e-01, -1.6956e-01,\n",
       "                       6.7200e-02, -1.2524e-01,  3.4106e-01,  2.8488e-02,  3.6743e-01,\n",
       "                      -3.5938e-01, -1.1432e-01,  1.2732e-01,  1.8234e-02,  1.8387e-02,\n",
       "                       4.7180e-02, -2.0679e-01,  7.2815e-02, -4.3408e-01, -7.1826e-01,\n",
       "                      -4.3066e-01,  1.4453e+00,  1.6187e-01,  3.2080e-01,  2.3792e-01,\n",
       "                       4.1724e-01, -3.7646e-01, -4.2786e-02, -6.2842e-01, -3.9380e-01,\n",
       "                      -8.4082e-01,  1.0065e-01, -5.0262e-02, -2.7271e-01, -2.0972e-01,\n",
       "                      -1.5356e-01, -3.2878e-04,  6.9971e-01, -2.0789e-01, -3.3911e-01,\n",
       "                       6.5186e-01, -4.9210e-03,  2.7435e-02,  2.0279e-02,  1.8265e-02,\n",
       "                      -7.5798e-03, -5.3444e-03,  5.1300e-02,  9.8724e-03,  2.6810e-02,\n",
       "                       2.0645e-02,  2.6062e-02, -1.2405e-02,  1.1505e-02, -1.6575e-03,\n",
       "                      -9.7351e-03, -1.0971e-02, -1.4963e-03, -1.2909e-02, -8.3847e-03,\n",
       "                      -4.9973e-03,  2.1255e-02,  3.7140e-02,  1.2115e-02, -2.1637e-02,\n",
       "                      -1.8677e-02, -9.7504e-03,  2.4872e-03,  2.2934e-02,  6.2256e-02,\n",
       "                      -1.8787e-03,  1.4992e-02, -1.1864e-02, -2.2797e-02,  1.4277e-03,\n",
       "                       9.3918e-03,  1.0101e-02,  8.5068e-03,  1.3138e-02,  1.3285e-03,\n",
       "                       1.2596e-02, -2.6360e-03, -9.4452e-03,  8.1329e-03,  9.1791e-04,\n",
       "                       6.4087e-03,  6.9046e-04,  1.5160e-02, -6.4011e-03,  1.6464e-02,\n",
       "                      -1.1902e-02, -2.3102e-02, -1.0757e-02, -1.0736e-01,  4.6997e-02,\n",
       "                      -1.1131e-02, -2.3209e-02,  2.6657e-02,  2.2583e-02,  1.0452e-02,\n",
       "                      -7.9727e-03,  1.8930e-03,  1.5038e-02,  2.2156e-02, -4.3983e-03,\n",
       "                       6.5674e-01, -2.0059e+00,  2.9004e+00,  5.2686e-01,  2.7852e+00,\n",
       "                      -6.5283e-01,  1.3623e+00, -7.1875e-01, -6.1279e-02, -9.6191e-01,\n",
       "                       2.2363e+00, -8.4912e-01,  1.9365e+00,  3.6206e-01, -1.7588e+00,\n",
       "                       2.1465e+00, -1.6152e+00, -2.5391e-01, -1.7510e+00,  1.3711e+00,\n",
       "                      -1.0801e+00, -1.1143e+00,  1.3008e+00, -1.5586e+00,  2.1942e-02,\n",
       "                       1.1224e-01,  9.0771e-01, -1.8564e+00, -3.2482e-03,  4.6445e+00,\n",
       "                       3.2871e+00, -4.4648e+00, -8.0762e-01,  1.3311e+00,  9.4336e-01,\n",
       "                       1.9512e+00, -4.7891e+00, -8.4814e-01,  5.9814e-01, -5.8740e-01,\n",
       "                       2.5000e+00,  1.3486e+00, -3.1934e-01, -2.6113e+00,  1.4463e+00,\n",
       "                      -4.6211e+00, -1.1908e-01,  4.3701e-01,  5.9375e-01,  3.9697e-01,\n",
       "                       6.9482e-01, -1.3438e+00, -2.1973e-01,  2.3730e+00, -4.0977e+00,\n",
       "                       1.1807e+00,  2.4785e+00,  2.1521e-01,  2.5757e-01,  1.0791e+00,\n",
       "                      -1.3984e+00,  1.2754e+00, -3.2288e-02,  1.2070e+00, -1.7158e+00,\n",
       "                      -1.9897e-01, -7.4609e-01,  4.3262e-01, -5.0690e-02,  2.5269e-01,\n",
       "                       1.5881e-01,  5.1331e-02, -3.0640e-01, -6.6748e-01,  3.5156e-02,\n",
       "                      -3.5858e-02,  1.9543e-01,  4.9400e-03,  8.3252e-02, -1.7761e-01,\n",
       "                      -2.5586e-01,  7.1289e-02,  1.6432e-03,  1.2659e-01, -6.8359e-02,\n",
       "                      -1.0748e-01,  5.8105e-02, -5.2344e-01,  2.2217e-01,  1.5594e-02,\n",
       "                      -3.2520e-01,  1.9128e-01, -3.5596e-01, -3.1128e-01, -3.6499e-01,\n",
       "                       1.4795e-01, -7.6355e-02, -7.7881e-02,  2.8979e-01, -3.3423e-01,\n",
       "                       7.7820e-02, -5.4443e-01,  1.0576e+00,  2.5952e-01,  2.3608e-01,\n",
       "                       1.7676e-01, -3.9502e-01, -9.0759e-02, -4.1455e-01,  1.9421e-01,\n",
       "                      -4.1675e-01,  3.9551e-01, -2.3218e-01,  3.4033e-01,  6.7822e-01,\n",
       "                       3.2495e-01, -1.8628e-01,  2.9266e-02,  1.6479e-02,  5.4541e-01,\n",
       "                       1.2793e-01, -5.7275e-01, -1.5735e-01,  3.6694e-01, -1.6312e-02,\n",
       "                      -3.5913e-01,  2.8125e-01,  4.0991e-01])),\n",
       "             ('transformer.h.2.self_attention.dense.weight',\n",
       "              tensor([[-0.0243,  0.0094, -0.0115,  ...,  0.0003,  0.0129,  0.0131],\n",
       "                      [ 0.0077,  0.0093,  0.0072,  ...,  0.0340, -0.0145,  0.0055],\n",
       "                      [ 0.0056,  0.0071, -0.0067,  ...,  0.0016, -0.0209,  0.0154],\n",
       "                      ...,\n",
       "                      [ 0.0320, -0.0011, -0.0424,  ..., -0.0059,  0.0089,  0.0066],\n",
       "                      [ 0.0017,  0.0122,  0.0141,  ..., -0.0023,  0.0226, -0.0064],\n",
       "                      [ 0.0047,  0.0085,  0.0081,  ..., -0.0081,  0.0046,  0.0015]])),\n",
       "             ('transformer.h.2.self_attention.dense.bias',\n",
       "              tensor([-0.0193, -0.0070, -0.0252,  ...,  0.0339,  0.0198, -0.0055])),\n",
       "             ('transformer.h.2.post_attention_layernorm.weight',\n",
       "              tensor([1.0537, 1.2871, 0.9834,  ..., 1.7959, 1.4951, 0.7256])),\n",
       "             ('transformer.h.2.post_attention_layernorm.bias',\n",
       "              tensor([ 0.1456, -0.1604, -0.1860,  ..., -0.3049, -0.1814, -0.2150])),\n",
       "             ('transformer.h.2.mlp.dense_h_to_4h.weight',\n",
       "              tensor([[ 0.0165,  0.0090, -0.0047,  ..., -0.0320, -0.0131,  0.0230],\n",
       "                      [-0.0115,  0.0277, -0.0008,  ...,  0.0133, -0.0173,  0.0277],\n",
       "                      [ 0.0121, -0.0254,  0.0207,  ...,  0.0071,  0.0124,  0.0153],\n",
       "                      ...,\n",
       "                      [ 0.0074,  0.0310,  0.0233,  ...,  0.0003, -0.0019, -0.0093],\n",
       "                      [ 0.0037, -0.0202,  0.0175,  ...,  0.0047,  0.0091,  0.0084],\n",
       "                      [-0.0027,  0.0007,  0.0194,  ..., -0.0055,  0.0210, -0.0006]])),\n",
       "             ('transformer.h.2.mlp.dense_h_to_4h.bias',\n",
       "              tensor([-0.5210, -0.4121, -0.0250,  ..., -0.1555, -0.1913, -0.0429])),\n",
       "             ('transformer.h.2.mlp.dense_4h_to_h.weight',\n",
       "              tensor([[-0.0060,  0.0004,  0.0009,  ..., -0.0034, -0.0008, -0.0070],\n",
       "                      [-0.0065, -0.0024,  0.0015,  ..., -0.0254, -0.0064, -0.0133],\n",
       "                      [ 0.0121,  0.0081, -0.0072,  ..., -0.0145,  0.0033, -0.0148],\n",
       "                      ...,\n",
       "                      [-0.0521, -0.0030,  0.0027,  ...,  0.0099,  0.0034,  0.0225],\n",
       "                      [-0.0119,  0.0185,  0.0021,  ..., -0.0035, -0.0348, -0.0259],\n",
       "                      [-0.0021, -0.0090, -0.0103,  ...,  0.0092, -0.0172, -0.0052]])),\n",
       "             ('transformer.h.2.mlp.dense_4h_to_h.bias',\n",
       "              tensor([ 0.0325, -0.0051,  0.0272,  ..., -0.1240,  0.0008, -0.0113])),\n",
       "             ('transformer.h.3.input_layernorm.weight',\n",
       "              tensor([1.0000, 1.2520, 0.9409,  ..., 1.6211, 1.3506, 0.7339])),\n",
       "             ('transformer.h.3.input_layernorm.bias',\n",
       "              tensor([-0.0795, -0.0817,  0.0488,  ..., -0.0072, -0.1219,  0.0293])),\n",
       "             ('transformer.h.3.self_attention.query_key_value.weight',\n",
       "              tensor([[ 0.0095,  0.0057, -0.0080,  ...,  0.0219, -0.0148, -0.0130],\n",
       "                      [ 0.0193, -0.0334,  0.0050,  ...,  0.0134, -0.0417, -0.0171],\n",
       "                      [-0.0163, -0.0403,  0.0039,  ..., -0.0402,  0.0236,  0.0016],\n",
       "                      ...,\n",
       "                      [-0.0063,  0.0120, -0.0151,  ..., -0.0148,  0.0224,  0.0094],\n",
       "                      [-0.0025,  0.0174, -0.0188,  ...,  0.0356, -0.0130,  0.0045],\n",
       "                      [ 0.0090, -0.0090, -0.0067,  ..., -0.0181,  0.0072, -0.0025]])),\n",
       "             ('transformer.h.3.self_attention.query_key_value.bias',\n",
       "              tensor([-1.1467e-02,  1.4236e-02,  2.5757e-02,  2.0809e-03, -1.2703e-02,\n",
       "                      -6.9275e-02,  4.5824e-04, -6.7078e-02,  4.1412e-02,  3.1128e-03,\n",
       "                      -1.3924e-02,  4.9561e-02, -2.8595e-02, -2.8168e-02, -3.2749e-03,\n",
       "                      -3.1982e-02, -2.1973e-02,  4.3152e-02,  1.0582e-02,  2.0462e-02,\n",
       "                       1.1414e-02, -2.7218e-03,  8.7646e-02, -3.2616e-03, -4.0703e-03,\n",
       "                      -1.1314e-02, -6.1493e-02,  5.2681e-03, -6.7078e-02,  2.9678e-02,\n",
       "                       2.3819e-02, -1.7700e-02,  2.1591e-02, -1.9623e-02, -2.1957e-02,\n",
       "                      -3.8025e-02, -3.7811e-02,  1.7532e-02, -7.3975e-02, -1.2922e-03,\n",
       "                       1.1742e-04, -9.0485e-03, -1.2383e-02,  4.6387e-02,  8.7280e-03,\n",
       "                      -2.7115e-02,  3.4302e-02,  1.5007e-02,  4.3671e-02,  2.2964e-03,\n",
       "                       4.5654e-02, -2.7466e-02,  1.5297e-02,  1.8250e-02,  3.0117e-03,\n",
       "                       5.6915e-02, -1.7487e-02,  1.2909e-02,  2.4139e-02,  5.5878e-02,\n",
       "                      -1.5732e-02, -1.0994e-02,  7.1678e-03,  2.1210e-02,  9.8267e-02,\n",
       "                      -3.2440e-02, -5.0537e-02, -1.9867e-02, -1.3586e-01, -5.5328e-02,\n",
       "                       1.3657e-03, -1.0620e-02,  1.1932e-01,  1.8359e-01, -6.5369e-02,\n",
       "                      -2.5269e-02, -4.1809e-03,  2.6484e+00, -2.0630e-01,  1.3586e-01,\n",
       "                      -6.6711e-02,  2.5772e-02,  6.6071e-03,  5.2147e-03, -4.9011e-02,\n",
       "                       8.7204e-03,  1.4143e-03, -2.0020e-01, -8.6792e-02, -2.6321e-02,\n",
       "                       9.5886e-02, -3.4027e-02, -7.7209e-02, -1.4026e-01,  1.9470e-02,\n",
       "                      -2.9358e-02,  9.1553e-02, -1.4453e-01, -6.2347e-02,  2.6440e-01,\n",
       "                       1.1902e-01, -6.2408e-03,  5.7007e-02,  5.7617e-02, -4.7607e-03,\n",
       "                       3.7451e-01,  1.5388e-02, -5.1056e-02, -2.4336e+00, -9.8389e-02,\n",
       "                       1.1615e-01,  7.6111e-02, -5.8327e-03, -5.8685e-02,  2.1008e-01,\n",
       "                       2.6154e-02, -8.5144e-02,  1.1368e-02, -4.6143e-01,  1.5976e-02,\n",
       "                      -2.5391e-02,  1.2611e-02, -1.4050e-01, -1.1560e-01, -2.6760e-03,\n",
       "                      -8.4961e-02,  1.2061e-01,  5.3314e-02, -3.7155e-03,  4.5471e-02,\n",
       "                       9.5154e-02, -2.5208e-02,  4.2175e-02,  1.2939e-02, -2.0844e-02,\n",
       "                      -2.3560e-02, -2.2629e-02, -6.7932e-02,  3.5065e-02, -2.8439e-03,\n",
       "                      -3.8696e-02, -4.3506e-01,  9.1919e-02,  7.4280e-02, -2.1313e-01,\n",
       "                      -5.0720e-02, -2.8976e-02, -2.6901e-02,  1.1938e-01,  1.2337e-02,\n",
       "                      -4.0253e-02,  4.6570e-02,  7.3181e-02, -3.6163e-03, -7.2693e-02,\n",
       "                      -9.6741e-02,  2.6932e-02, -3.2867e-02,  5.2567e-03,  3.6957e-02,\n",
       "                       1.1047e-02, -1.0394e-01,  4.2358e-02,  1.2573e-02,  1.3367e-01,\n",
       "                       5.2612e-02, -8.1360e-02, -5.0903e-02, -2.3645e-01, -8.5327e-02,\n",
       "                      -7.0007e-02, -1.1772e-02,  3.8550e-01, -6.8420e-02,  4.2267e-02,\n",
       "                      -5.2277e-02,  8.8501e-03,  4.9255e-02, -3.8391e-02,  7.5256e-02,\n",
       "                       4.4441e-03,  1.0034e-01,  8.3008e-02, -2.3972e-02, -3.0098e-03,\n",
       "                       8.9539e-02, -1.2311e-01, -4.2297e-02, -9.3445e-02, -3.9673e-02,\n",
       "                       7.8247e-02, -3.1952e-02,  6.8848e-02,  5.8929e-02,  3.5840e-01,\n",
       "                      -2.6514e-01,  1.4014e-01,  4.9530e-02,  2.3279e-01,  1.6333e-01,\n",
       "                      -1.6528e-01, -2.8979e-01, -2.5436e-02, -5.2539e-01, -1.7322e-01,\n",
       "                      -4.6883e-03,  6.3843e-02, -1.0193e-01,  5.1709e-01, -1.2903e-01,\n",
       "                      -7.2754e-02,  1.1340e-01,  1.1920e-01,  2.2046e-01,  7.3059e-02,\n",
       "                      -8.8013e-02,  3.1396e-01,  3.5913e-01, -7.8003e-02, -1.7044e-02,\n",
       "                      -5.1953e-01,  3.9941e-01,  3.0542e-01,  2.4323e-02, -3.0957e-01,\n",
       "                       6.3660e-02, -9.8938e-02, -4.7510e-01,  1.1694e-01,  2.5806e-01,\n",
       "                      -3.9136e-01, -7.8857e-02,  2.6050e-01,  1.3025e-01, -1.9690e-01,\n",
       "                      -2.1406e+00, -2.6289e+00, -6.0364e-02, -1.6626e-01,  2.4463e-01,\n",
       "                      -1.3367e-01,  8.4381e-03, -1.5466e-01,  1.7426e-02,  1.9324e-01,\n",
       "                       1.4725e-02, -5.2063e-02, -5.6445e-01, -1.2764e+00,  1.7407e-01,\n",
       "                      -1.9385e-01, -1.9397e-01,  2.5244e-01,  2.4646e-01,  2.7686e-01,\n",
       "                      -5.8670e-03,  1.0277e-02, -7.2403e-03, -7.7515e-03,  1.1757e-02,\n",
       "                       2.7542e-02, -2.1704e-01, -2.2217e-02,  3.1586e-02, -4.0245e-03,\n",
       "                      -8.2703e-03, -1.0536e-02,  6.9771e-03,  7.9651e-03, -8.3466e-03,\n",
       "                       5.0964e-03, -1.6571e-02, -1.4282e-02, -5.2032e-03,  2.4300e-03,\n",
       "                      -2.2583e-02,  4.5624e-03,  2.8122e-02, -4.7760e-03, -3.8967e-03,\n",
       "                       4.3344e-04,  5.7564e-03, -9.6560e-04,  5.2277e-02,  1.4832e-02,\n",
       "                       8.2474e-03, -1.2939e-02,  3.5828e-02, -6.8426e-04,  3.2745e-02,\n",
       "                      -2.0180e-03,  2.1484e-02,  2.0432e-02,  3.3016e-03, -7.8278e-03,\n",
       "                       5.0977e-01,  3.7441e-03, -3.7903e-02,  7.3090e-03,  1.0597e-02,\n",
       "                       6.1951e-02, -1.7163e-01, -5.3062e-03,  2.1057e-02,  1.6525e-02,\n",
       "                      -3.2440e-02,  6.2866e-03, -1.3298e-02, -2.1683e-02, -7.2632e-02,\n",
       "                      -2.9251e-02, -1.3977e-02,  3.7231e-02,  1.2123e-02, -3.6499e-02,\n",
       "                       2.1572e-03, -7.3471e-03,  9.9411e-03, -9.5825e-03, -2.9572e-02,\n",
       "                       3.2397e-01,  2.6566e-02,  3.5215e+00, -2.8491e-04, -8.7891e-01,\n",
       "                       2.5439e-01, -1.4990e-01, -1.0684e+00, -6.9336e-01, -7.9224e-02,\n",
       "                      -4.2017e-01, -8.5327e-02, -2.0703e-01, -6.4575e-02,  4.2725e-01,\n",
       "                      -5.9570e-01,  6.4062e-01, -8.5742e-01,  1.2720e-01,  7.2314e-01,\n",
       "                       1.5083e-02,  2.1594e-01,  9.6436e-01,  2.4582e-02, -2.1851e-01,\n",
       "                      -3.6963e-01,  1.3039e-02, -5.0842e-02, -8.7939e-01,  3.4937e-01,\n",
       "                       7.5098e-01,  4.8755e-01,  9.6924e-01,  5.6738e-01,  7.2815e-02,\n",
       "                       5.6592e-01,  3.1006e-01, -5.6445e-01,  7.9541e-01, -5.2686e-01,\n",
       "                      -1.6138e-01, -8.3350e-01,  9.3140e-02,  5.5542e-02, -8.3203e-01,\n",
       "                      -8.7939e-01,  8.4082e-01,  2.2832e+00, -6.7920e-01, -6.6956e-02,\n",
       "                       3.3867e+00, -2.6758e-01, -2.0844e-02,  5.3467e-01, -1.3440e-01,\n",
       "                      -7.1924e-01, -7.8760e-01,  1.3086e-01, -3.7378e-01,  3.5791e-01,\n",
       "                       1.6809e-01, -4.2407e-01,  1.0234e+00,  3.2300e-01, -1.5205e-02,\n",
       "                      -3.3936e-02,  1.7792e-02, -5.1498e-04,  8.7280e-03, -4.7150e-03,\n",
       "                       8.1940e-03,  2.7496e-02,  4.3304e-02,  5.7144e-03, -1.7471e-02,\n",
       "                      -2.9999e-02, -5.1117e-02, -1.0582e-02,  2.1957e-02, -4.9133e-02,\n",
       "                      -6.0425e-02, -1.3054e-02,  1.5717e-02,  3.7659e-02, -9.3231e-03,\n",
       "                      -4.5929e-02,  2.0050e-02,  2.3575e-02, -1.2489e-02,  4.1443e-02,\n",
       "                       5.5176e-02,  2.6184e-02,  1.8112e-02, -1.6357e-02, -9.6054e-03,\n",
       "                      -1.7899e-02, -3.8719e-03,  1.4893e-02,  1.8387e-02,  3.2673e-03,\n",
       "                       5.2605e-03,  5.0163e-03, -1.1955e-02,  2.1042e-02, -8.6121e-02,\n",
       "                       6.6406e-02, -4.1870e-02, -4.1046e-03, -6.2164e-02, -3.7415e-02,\n",
       "                      -2.9434e-02,  3.3234e-02,  2.5970e-02, -4.3182e-02, -2.5986e-02,\n",
       "                      -9.4032e-04,  4.3602e-03,  1.9165e-02, -6.2065e-03, -1.0910e-02,\n",
       "                      -2.8946e-02,  5.8380e-02,  5.3787e-03,  7.4402e-02,  1.0399e-02,\n",
       "                       2.8854e-02, -1.4565e-02, -2.8244e-02,  4.9829e-01, -4.9194e-01,\n",
       "                       6.9629e-01, -8.3496e-01,  1.0576e+00,  1.4307e+00,  1.2549e-01,\n",
       "                       4.4775e-01, -3.7646e-01, -2.3303e-01, -6.0889e-01, -3.4351e-01,\n",
       "                      -6.1646e-02,  4.5654e-01, -2.8174e-01,  1.0049e+00, -3.7915e-01,\n",
       "                      -1.0840e-01,  8.9014e-01, -3.1616e-01,  2.0293e+00, -2.0813e-01,\n",
       "                      -2.4182e-01, -6.5137e-01, -4.4165e-01,  3.8281e-01,  1.3975e+00,\n",
       "                       2.6443e-02, -5.4150e-01,  4.8779e-01,  6.3086e-01,  2.1643e-01,\n",
       "                      -7.2754e-01,  1.6768e+00, -3.3179e-01, -2.6514e-01, -2.7168e+00,\n",
       "                       5.0391e-01, -2.4878e-01, -1.3008e+00, -2.0742e+00,  2.4658e-01,\n",
       "                      -3.6597e-01,  2.3352e-01,  4.5776e-01,  1.9250e-01, -3.2288e-02,\n",
       "                      -1.7246e+00, -1.6174e-01,  3.8662e-03,  6.4014e-01,  2.9629e+00,\n",
       "                       2.8730e+00, -1.1562e+00, -2.7661e-01, -1.0430e+00,  2.1591e-02,\n",
       "                      -7.5146e-01, -5.3320e-01,  2.2791e-01,  8.4814e-01,  7.3633e-01,\n",
       "                      -8.1885e-01,  4.5532e-01,  1.1603e-01,  2.4597e-02, -3.9844e-01,\n",
       "                      -2.1484e-01, -4.2773e-01,  9.3848e-01,  1.7029e-01,  6.5674e-01,\n",
       "                       1.1330e-02, -2.1924e-01, -2.2018e-02,  1.7944e-01,  6.3281e-01,\n",
       "                      -1.8018e-01,  2.6025e-01,  3.5425e-01, -1.0582e-02, -2.0764e-01,\n",
       "                       2.0850e-01,  2.1835e-02,  6.0352e-01,  7.0068e-01, -6.4087e-02,\n",
       "                      -1.0388e-01,  4.5239e-01, -3.3203e-01,  1.9495e-01, -1.7065e-01,\n",
       "                      -2.9346e-01,  1.9739e-01,  7.9041e-02, -5.8929e-02,  1.8469e-01,\n",
       "                      -2.4738e-03, -1.7407e-01, -5.7343e-02,  3.1738e-01,  1.3135e+00,\n",
       "                      -1.0114e-01, -1.8252e+00,  4.5349e-02, -5.4932e-02, -7.2266e-02,\n",
       "                      -1.0187e-01,  1.3770e-01, -2.5952e-01,  1.5784e-01,  2.3157e-01,\n",
       "                      -2.3938e-01, -2.3511e-01, -9.9805e-01,  3.6304e-01,  9.1614e-02,\n",
       "                      -2.5024e-02, -2.2180e-01, -1.1066e-01,  1.5125e-01, -3.2300e-01,\n",
       "                       1.3037e-01,  1.4648e-01,  6.3599e-02, -2.5063e-03,  5.0928e-01,\n",
       "                       3.9703e-02, -1.9623e-02,  4.6349e-03,  5.2032e-02,  1.5526e-02,\n",
       "                      -1.5778e-02, -1.9714e-02,  1.4496e-02,  1.7807e-02,  1.6373e-02,\n",
       "                      -5.8174e-04,  1.3664e-02,  1.0719e-02, -6.5193e-03, -8.4686e-04,\n",
       "                      -2.7786e-02,  6.5079e-03, -1.4931e-02,  1.2541e-03,  1.8753e-02,\n",
       "                      -5.8784e-03,  1.9855e-03, -1.9703e-03,  2.4433e-03, -3.6979e-04,\n",
       "                       4.2963e-04, -9.3765e-03, -1.9608e-02, -6.3477e-03, -1.8280e-02,\n",
       "                      -2.2507e-02,  2.2293e-02,  1.1391e-02,  3.4637e-02,  9.7275e-03,\n",
       "                       2.5208e-02,  3.9978e-03,  2.6703e-02,  6.0394e-02, -1.5388e-02,\n",
       "                       1.7853e-02, -2.1744e-02,  1.8539e-02, -3.0632e-03, -3.1924e-04,\n",
       "                      -6.9580e-03, -9.9792e-03,  3.1185e-03, -9.4376e-03,  8.9798e-03,\n",
       "                       7.6752e-03,  2.2449e-01,  8.9407e-04, -2.4261e-02,  8.9645e-04,\n",
       "                      -4.5586e-03, -4.8332e-03,  2.2095e-02, -1.4114e-02, -2.2131e-01,\n",
       "                       1.8179e-04,  1.6190e-02,  4.2847e-02, -2.2171e-02,  6.4850e-03,\n",
       "                       7.8271e-01, -9.2383e-01, -5.5762e-01,  1.5149e-01, -1.7676e-01,\n",
       "                       3.4888e-01, -6.2744e-01,  2.9102e-01, -4.9835e-02,  2.0935e-01,\n",
       "                      -1.8008e+00,  1.4121e+00,  8.1970e-02, -8.1299e-01,  8.1445e-01,\n",
       "                       1.7883e-01,  3.3838e-01,  2.2119e-01,  4.8291e-01, -1.0925e-01,\n",
       "                      -1.1432e-01, -1.5735e-01,  4.1772e-01,  6.7383e-01, -7.0166e-01,\n",
       "                      -7.4219e-01, -1.2115e-01, -3.2495e-01,  3.0151e-01, -6.1230e-01,\n",
       "                      -6.5869e-01, -6.5967e-01,  3.1104e-01, -7.0557e-01,  5.8594e-01,\n",
       "                       2.2534e-01,  6.0352e-01, -6.2158e-01, -5.1367e-01,  2.7783e-01,\n",
       "                      -9.1003e-02, -1.9238e-01,  6.3428e-01,  4.0381e-01,  1.0586e+00,\n",
       "                       7.8125e-02,  2.7124e-01, -2.4048e-01,  1.9946e-01, -4.6582e-01,\n",
       "                      -2.0984e-01, -1.7725e+00,  3.9673e-02,  1.0178e-02,  9.7705e-01,\n",
       "                      -9.1125e-02, -1.2817e-02,  1.3123e-02, -4.0381e-01,  1.9287e-01,\n",
       "                      -3.9355e-01, -1.2976e-01,  1.0400e+00,  8.1250e-01, -5.7564e-03,\n",
       "                      -3.3783e-02,  2.2049e-03, -2.7847e-03,  9.2224e-02, -3.4149e-02,\n",
       "                       4.2816e-02, -5.2002e-02, -1.5808e-02, -3.7659e-02,  2.3468e-02,\n",
       "                      -1.2527e-02, -2.9545e-03, -1.0910e-02, -6.4514e-02, -1.3666e-03,\n",
       "                      -2.9590e-01,  4.6425e-03,  7.8918e-02,  5.1880e-03,  1.3916e-02,\n",
       "                      -3.7207e-01,  6.8855e-04, -2.6016e-02,  3.4761e-04, -1.4353e-03,\n",
       "                      -1.2708e-01,  1.5049e-03,  6.3019e-03, -3.3752e-02,  1.2100e-02,\n",
       "                       2.3895e-02,  2.3270e-03, -5.5237e-03, -3.1376e-03, -2.0715e-01,\n",
       "                      -2.5787e-02, -1.0681e-02,  2.4078e-02, -1.4931e-02, -7.0229e-03,\n",
       "                       2.1782e-03,  3.9825e-02,  1.7181e-02, -1.8814e-02,  2.5986e-02,\n",
       "                       2.6428e-02,  2.9541e-02, -1.0010e-02, -3.3966e-02,  7.9468e-02,\n",
       "                      -3.9337e-02,  5.3101e-02, -1.6708e-02,  1.2932e-02, -1.9207e-03,\n",
       "                      -7.3303e-02,  9.2163e-03,  7.4692e-03, -1.6785e-02,  1.3908e-02,\n",
       "                      -3.4375e-01, -1.5662e-01, -7.3280e-03])),\n",
       "             ('transformer.h.3.self_attention.dense.weight',\n",
       "              tensor([[ 6.1646e-03,  4.7798e-03,  2.1408e-02,  ...,  8.5068e-03,\n",
       "                       -1.8173e-02,  6.0387e-03],\n",
       "                      [-1.7838e-02,  2.9587e-02, -5.3787e-04,  ...,  9.5978e-03,\n",
       "                        2.0569e-02,  6.5651e-03],\n",
       "                      [-1.3191e-02, -4.7760e-03, -9.7454e-05,  ..., -3.2787e-03,\n",
       "                        1.2939e-02, -9.2411e-04],\n",
       "                      ...,\n",
       "                      [ 1.3763e-02, -1.8265e-02,  1.2407e-03,  ...,  2.4700e-03,\n",
       "                       -2.7267e-02, -1.2428e-02],\n",
       "                      [-1.1009e-02,  1.6708e-02, -1.4046e-02,  ..., -3.3264e-02,\n",
       "                       -2.0630e-02,  1.1742e-02],\n",
       "                      [ 7.7248e-03,  2.8801e-03,  1.1246e-02,  ..., -7.8201e-03,\n",
       "                        1.7700e-02,  1.0582e-02]])),\n",
       "             ('transformer.h.3.self_attention.dense.bias',\n",
       "              tensor([ 0.0188, -0.0357, -0.0086,  ..., -0.1084, -0.0207,  0.0088])),\n",
       "             ('transformer.h.3.post_attention_layernorm.weight',\n",
       "              tensor([1.1328, 1.3623, 1.0703,  ..., 1.7676, 1.5254, 0.7695])),\n",
       "             ('transformer.h.3.post_attention_layernorm.bias',\n",
       "              tensor([ 0.1631, -0.1713, -0.1874,  ..., -0.1115, -0.2029, -0.0848])),\n",
       "             ('transformer.h.3.mlp.dense_h_to_4h.weight',\n",
       "              tensor([[-3.1235e-02,  5.3558e-03,  6.1750e-05,  ..., -4.0207e-03,\n",
       "                        1.1375e-02,  1.0063e-02],\n",
       "                      [-2.4204e-03,  2.8641e-02,  1.8883e-03,  ..., -1.4557e-02,\n",
       "                       -2.5513e-02, -4.7455e-03],\n",
       "                      [-4.9877e-04, -1.6998e-02,  1.1539e-03,  ...,  1.1536e-02,\n",
       "                        8.3771e-03, -2.4414e-02],\n",
       "                      ...,\n",
       "                      [-1.6022e-02,  1.9943e-02, -4.6234e-03,  ...,  9.6436e-03,\n",
       "                       -1.1108e-02,  4.1290e-02],\n",
       "                      [ 2.5024e-02, -1.7090e-02,  3.3360e-03,  ..., -2.7802e-02,\n",
       "                        9.8877e-03, -2.9114e-02],\n",
       "                      [-2.0142e-02,  2.1423e-02, -3.0651e-03,  ...,  1.2535e-02,\n",
       "                        1.3420e-02,  1.7075e-02]])),\n",
       "             ('transformer.h.3.mlp.dense_h_to_4h.bias',\n",
       "              tensor([-0.0118, -0.0833, -0.3960,  ..., -0.4927, -0.1273, -0.0334])),\n",
       "             ('transformer.h.3.mlp.dense_4h_to_h.weight',\n",
       "              tensor([[ 0.0109,  0.0019,  0.0193,  ...,  0.0006, -0.0140,  0.0323],\n",
       "                      [-0.0131, -0.0111,  0.0032,  ...,  0.0168,  0.0210, -0.0180],\n",
       "                      [-0.0211,  0.0305, -0.0129,  ...,  0.0130,  0.0047,  0.0021],\n",
       "                      ...,\n",
       "                      [ 0.0028,  0.0174,  0.0239,  ..., -0.0093,  0.0012, -0.0073],\n",
       "                      [ 0.0002, -0.0004, -0.0005,  ..., -0.0004, -0.0122,  0.0025],\n",
       "                      [ 0.0078,  0.0124, -0.0135,  ...,  0.0094,  0.0254, -0.0125]])),\n",
       "             ('transformer.h.3.mlp.dense_4h_to_h.bias',\n",
       "              tensor([ 0.0168, -0.0649,  0.0106,  ..., -0.1026, -0.0345, -0.0318])),\n",
       "             ('transformer.h.4.input_layernorm.weight',\n",
       "              tensor([1.0098, 1.2373, 0.8921,  ..., 1.6436, 1.3389, 0.7031])),\n",
       "             ('transformer.h.4.input_layernorm.bias',\n",
       "              tensor([-0.0637, -0.0417,  0.0852,  ..., -0.0303, -0.1237,  0.0047])),\n",
       "             ('transformer.h.4.self_attention.query_key_value.weight',\n",
       "              tensor([[-0.0062,  0.0083,  0.0172,  ..., -0.0264, -0.0144, -0.0039],\n",
       "                      [-0.0151,  0.0220,  0.0400,  ..., -0.0158, -0.0055,  0.0199],\n",
       "                      [-0.0093,  0.0046,  0.0043,  ...,  0.0242, -0.0175, -0.0562],\n",
       "                      ...,\n",
       "                      [ 0.0071, -0.0137,  0.0027,  ...,  0.0048,  0.0034,  0.0093],\n",
       "                      [ 0.0180, -0.0352, -0.0046,  ..., -0.0079, -0.0320, -0.0016],\n",
       "                      [-0.0016,  0.0022, -0.0115,  ..., -0.0105, -0.0121,  0.0272]])),\n",
       "             ('transformer.h.4.self_attention.query_key_value.bias',\n",
       "              tensor([-2.7237e-02, -1.2779e-02, -2.5349e-03, -3.8666e-02,  1.2665e-02,\n",
       "                       2.4384e-02,  8.8806e-03,  6.6414e-03, -1.8967e-02,  3.0136e-02,\n",
       "                      -1.6006e-02,  1.9714e-02, -1.0094e-02, -1.8127e-02,  1.1932e-02,\n",
       "                       1.6541e-02,  3.7598e-02,  1.1490e-02,  2.5650e-02, -9.6054e-03,\n",
       "                      -2.9297e-03, -1.6296e-02,  5.0476e-02,  2.5375e-02,  4.3701e-02,\n",
       "                      -2.5620e-02,  3.9276e-02,  2.7954e-02, -3.0685e-02,  1.0864e-02,\n",
       "                       2.3254e-02,  7.9651e-03,  3.1219e-02, -2.8015e-02, -2.9358e-02,\n",
       "                       3.0670e-02, -2.6642e-02, -8.0505e-02,  2.1912e-02,  1.4084e-02,\n",
       "                       8.0872e-03,  5.9998e-02,  3.0396e-02, -5.1737e-04,  1.6079e-03,\n",
       "                      -1.5503e-02, -2.3849e-02,  2.4357e-03,  1.1215e-02,  5.6580e-02,\n",
       "                       3.3386e-02,  1.9226e-03, -4.6326e-02,  2.6417e-03,  2.8595e-02,\n",
       "                       3.1555e-02, -3.5187e-02,  4.7722e-03, -2.0996e-02,  7.5645e-03,\n",
       "                      -2.8076e-03,  3.7048e-02,  2.6321e-02, -4.9805e-02,  5.5481e-02,\n",
       "                       3.5248e-02, -3.9787e-03, -3.6645e-04,  3.1219e-02, -1.2598e-01,\n",
       "                       4.1504e-02,  5.0018e-02, -2.1210e-02,  4.4580e-01,  2.9007e-02,\n",
       "                       2.0737e-02, -3.9551e-02, -2.9373e-02, -3.3691e-02, -5.7831e-02,\n",
       "                       3.0945e-02,  3.7231e-02, -3.0350e-02,  3.2867e-02, -6.5002e-02,\n",
       "                      -4.2358e-02, -1.2245e-02,  7.8812e-03, -4.1534e-02,  3.8055e-02,\n",
       "                       3.1592e-01, -1.6309e-01, -2.7298e-02,  8.4534e-02, -8.0688e-02,\n",
       "                      -1.0612e-02,  8.2397e-03, -2.3254e-02, -5.9204e-02,  2.6016e-02,\n",
       "                       7.0557e-02,  2.2278e-02,  6.8909e-02,  1.1444e-01,  1.5121e-02,\n",
       "                      -1.2039e-02,  9.4360e-02,  3.7231e-02, -3.3752e-02,  3.0991e-02,\n",
       "                       1.2402e-01, -2.9388e-02,  7.0740e-02,  7.0251e-02, -2.2491e-02,\n",
       "                       2.4414e-02,  2.3941e-02,  5.5573e-02,  4.2267e-02,  1.3660e-01,\n",
       "                      -1.2085e-01,  3.1433e-02,  3.9612e-02, -5.7144e-03, -3.1708e-02,\n",
       "                       7.2205e-02, -5.1331e-02,  3.1830e-02, -2.9037e-02,  1.0651e-02,\n",
       "                      -2.9144e-02, -2.1248e-03, -1.0872e-02,  1.9214e-01,  3.8471e-03,\n",
       "                       1.4236e-02,  4.7607e-02, -1.5114e-02,  2.1805e-02,  3.9703e-02,\n",
       "                       1.0414e-02, -1.1635e-02,  2.5421e-02,  1.0162e-02, -2.6798e-03,\n",
       "                       8.1587e-04,  1.5038e-02,  1.6907e-02, -5.2719e-03,  1.5961e-02,\n",
       "                      -2.6291e-02,  9.9945e-03, -2.9419e-02, -2.3972e-02, -2.5742e-02,\n",
       "                       1.1188e-01,  2.1500e-02,  3.2349e-02,  1.0269e-02,  3.1204e-02,\n",
       "                      -7.8278e-03,  3.3325e-02,  9.7961e-03, -1.8339e-03, -1.6891e-02,\n",
       "                       1.5900e-02, -4.1199e-02, -6.6681e-03,  2.4857e-02, -1.5839e-02,\n",
       "                       5.6000e-03, -7.5769e-04,  4.1107e-02, -8.0643e-03,  2.6703e-02,\n",
       "                      -7.8888e-03,  3.6163e-03,  1.5762e-02, -2.1606e-02,  6.1836e-03,\n",
       "                      -9.9106e-03, -2.5055e-02,  3.7811e-02, -1.0662e-03,  1.2878e-02,\n",
       "                       5.0011e-03,  1.8448e-02,  1.2177e-02,  6.1615e-02, -6.2714e-03,\n",
       "                      -2.0279e-02, -2.4395e-03, -7.6904e-01, -3.6865e-02, -2.0432e-02,\n",
       "                       1.2964e-01, -6.5125e-02, -1.7639e-01, -1.9373e-01, -3.8086e-02,\n",
       "                      -1.2856e-02, -9.1064e-02, -2.8516e-01,  2.5543e-02,  4.1771e-03,\n",
       "                      -4.7363e-02,  2.2961e-01, -1.1938e-01,  4.2114e-01, -1.3323e-03,\n",
       "                      -2.3819e-02,  3.9941e-01, -7.5073e-02, -1.1924e+00, -2.0374e-01,\n",
       "                      -7.5760e-03, -1.6675e-01, -2.4062e+00, -5.0323e-02,  1.7041e-01,\n",
       "                       4.8096e-02,  3.3740e-01,  1.3574e-01, -2.1875e-01,  6.0889e-01,\n",
       "                       1.5942e-01,  1.4932e+00,  2.9346e-01,  9.6191e-02,  1.2585e-01,\n",
       "                       3.0884e-01, -1.0205e-01, -7.7209e-02,  5.0232e-02, -5.0732e-01,\n",
       "                       2.3865e-01,  1.6284e-01, -7.3608e-02, -9.1003e-02,  2.0129e-01,\n",
       "                       4.6906e-02, -2.5195e-01,  3.8281e-01, -8.6060e-02, -1.1307e-02,\n",
       "                      -5.1178e-02, -1.1774e-01, -9.4666e-02, -7.9041e-02, -1.2238e-01,\n",
       "                       2.0227e-01, -1.1780e-01, -5.3986e-02,  9.6875e-01, -4.9683e-01,\n",
       "                      -2.1240e-01, -6.8893e-03, -1.0109e-03, -1.2009e-02,  4.1046e-03,\n",
       "                       9.2316e-03, -3.1281e-02,  2.9144e-03, -3.7445e-02,  3.3081e-02,\n",
       "                       2.0561e-03,  1.6724e-02, -3.7750e-02, -8.1604e-02, -6.8054e-03,\n",
       "                      -1.5884e-02,  2.6520e-02, -3.0624e-02,  2.4185e-02,  1.6556e-02,\n",
       "                      -4.1016e-02,  2.2446e-02, -1.9958e-02,  4.8004e-02, -6.6566e-03,\n",
       "                       5.6953e-03, -1.3344e-02,  1.7185e-03,  4.4067e-02,  1.6689e-03,\n",
       "                      -1.6495e-02, -9.8648e-03,  2.5803e-02,  1.7990e-02,  2.3071e-02,\n",
       "                       6.1569e-03, -3.9253e-03, -3.2928e-02, -3.9139e-03, -4.6509e-02,\n",
       "                      -2.4719e-02, -1.9394e-02,  4.3066e-01,  7.0038e-03,  2.6596e-02,\n",
       "                       6.8703e-03, -4.1275e-03,  6.2141e-03,  1.7944e-02,  5.4817e-03,\n",
       "                      -3.1829e-05,  9.1887e-04,  3.0609e-02, -2.9663e-02, -7.8201e-03,\n",
       "                      -1.8570e-02,  1.7624e-02, -1.8677e-02, -1.1894e-02,  1.8570e-02,\n",
       "                      -1.5114e-02,  1.0361e-02,  2.6230e-02, -1.0590e-02, -2.3479e-03,\n",
       "                      -1.4748e-02, -1.6312e-02,  5.7840e-04,  1.7548e-02,  4.2450e-02,\n",
       "                       1.5152e-02,  3.5339e-02, -1.2341e-03, -2.7084e-02,  5.2490e-03,\n",
       "                      -1.8661e-02, -1.6006e-02, -3.5431e-02,  7.5264e-03,  4.2076e-03,\n",
       "                      -1.1187e-03, -2.9724e-02, -1.6464e-02, -2.7981e-03,  3.2349e-02,\n",
       "                       3.5534e-03,  3.0029e-02,  5.4779e-02,  1.8433e-02,  2.7908e-02,\n",
       "                       3.0670e-02, -2.2125e-02,  5.9776e-03, -2.1423e-02,  2.1652e-02,\n",
       "                      -2.4676e-04, -3.7651e-03,  2.2011e-03,  1.9058e-02, -1.2253e-02,\n",
       "                      -3.2196e-03, -1.5335e-02,  4.1107e-02,  1.4717e-02,  2.9956e-01,\n",
       "                      -2.7039e-02,  2.2644e-02,  1.5244e-02, -1.9455e-02, -1.1871e-02,\n",
       "                       7.1228e-02, -2.1164e-02, -4.9774e-02,  2.3331e-02, -1.9424e-02,\n",
       "                      -1.0567e-02, -1.0567e-02, -7.5951e-03,  2.7039e-02, -1.1566e-02,\n",
       "                       9.8572e-03, -2.8931e-02, -1.0956e-02, -5.8861e-03, -8.5144e-03,\n",
       "                      -2.2156e-02, -4.4891e-02, -8.4229e-03,  1.8082e-02, -5.1465e-01,\n",
       "                       2.5467e-02, -5.9586e-03,  2.0218e-02, -2.8052e-01,  2.5171e-01,\n",
       "                       4.9048e-01,  3.6652e-02,  1.2683e-01,  1.4587e-01, -2.3727e-02,\n",
       "                       1.5615e+00, -4.6295e-02, -5.9326e-01,  9.2285e-02, -2.5317e-01,\n",
       "                      -2.6440e-01, -2.1130e-01, -1.6504e+00,  4.2480e-01, -8.1909e-02,\n",
       "                      -2.1216e-01,  4.5630e-01, -2.9175e-01,  3.1174e-02,  5.2063e-02,\n",
       "                       4.3237e-01,  2.3621e-01,  4.0308e-01, -1.0010e-01,  2.0959e-01,\n",
       "                       8.3130e-02, -3.5156e-02,  2.0569e-01,  2.4451e-01, -3.5889e-01,\n",
       "                       5.8258e-02, -4.4995e-01, -1.1316e-01,  5.8929e-02,  6.0693e-01,\n",
       "                      -9.5361e-01,  1.4429e-01, -1.6089e-01, -3.5181e-01,  7.3975e-02,\n",
       "                       1.7505e-01,  5.7080e-01,  4.1931e-02, -7.7515e-02,  5.8990e-02,\n",
       "                      -1.2573e-01,  4.6997e-03, -1.7883e-02,  9.1309e-02, -1.7590e-01,\n",
       "                       2.8345e-01, -1.5060e-02,  2.0157e-02,  2.4072e-01, -2.0691e-01,\n",
       "                      -2.3499e-02, -2.4023e-01, -3.9697e-01, -2.6245e-01, -3.2666e-01,\n",
       "                       1.2091e-01,  3.6694e-01, -6.1157e-02,  3.9746e-01, -5.8603e-04,\n",
       "                       4.3823e-01,  4.0558e-02, -4.5630e-01,  5.1172e-01,  2.6191e+00,\n",
       "                      -2.2205e-01, -2.5928e-01, -1.9458e-01,  1.8665e-01, -1.8909e-01,\n",
       "                      -3.5107e-05, -2.8828e+00,  4.3823e-01,  5.6934e-01,  8.1177e-02,\n",
       "                      -8.5938e-02, -3.2056e-01,  6.2256e-02,  2.2534e-01,  5.7617e-01,\n",
       "                      -7.5146e-01,  3.1982e-01, -1.7285e-01, -3.5938e-01,  3.2410e-02,\n",
       "                      -2.3877e-01,  3.9612e-02, -2.5073e-01, -1.8591e-01, -2.5879e-01,\n",
       "                      -4.1962e-02, -4.6234e-02,  3.4766e-01, -1.4673e-01, -1.1562e+00,\n",
       "                      -1.3025e-01,  1.3893e-02, -3.7427e-01,  1.2854e-01,  1.2347e-01,\n",
       "                       3.3789e-01,  4.6606e-01,  8.1909e-02,  6.5527e-01,  4.1077e-02,\n",
       "                       2.0923e-01, -2.8882e-01, -1.1481e-01, -1.1652e-01,  7.9651e-03,\n",
       "                      -4.8413e-01, -3.5254e-01,  2.5903e-01, -1.6028e-01, -2.9761e-01,\n",
       "                      -1.3916e-01,  1.5735e-01, -2.8369e-01,  1.7334e-01,  7.8369e-02,\n",
       "                       1.6431e-01, -6.4746e-01,  5.3125e-01, -1.5979e-01,  7.3438e-01,\n",
       "                      -3.1909e-01, -4.9713e-02, -1.4038e-01,  1.5869e-01, -2.9858e-01,\n",
       "                      -3.9154e-02,  3.5059e-01,  1.1786e-01,  3.3521e-01, -1.1816e-01,\n",
       "                       1.2910e+00, -1.8555e+00, -6.9141e-01, -6.6406e-02, -4.2798e-01,\n",
       "                      -5.1611e-01,  3.1891e-02, -3.8795e-03, -2.4719e-01, -1.0773e-02,\n",
       "                       3.2318e-02,  3.0563e-02,  1.1176e-01,  3.2935e-01, -3.4961e-01,\n",
       "                      -9.0088e-01,  5.2002e-01,  3.3374e-01, -1.9751e-01,  5.2783e-01,\n",
       "                      -2.9102e-01, -1.5479e-01, -2.3193e-02, -1.2634e-01,  6.1584e-02,\n",
       "                      -7.4609e-01, -1.0480e-01,  6.6650e-02, -4.7778e-01,  1.0040e-01,\n",
       "                       2.6001e-02, -4.4556e-02, -1.2512e-01,  3.4741e-01, -4.3359e-01,\n",
       "                       2.2827e-02,  2.0569e-01, -2.0947e-01,  1.4368e-01,  1.8906e-02,\n",
       "                       3.2593e-02, -2.4792e-01, -1.6003e-01,  1.0803e-01,  6.1676e-02,\n",
       "                       1.5161e-01,  8.0713e-01,  4.4336e-01, -5.1709e-01, -1.7102e-01,\n",
       "                      -3.2886e-01, -7.7148e-02,  9.6680e-02,  1.0333e-01,  5.3680e-02,\n",
       "                      -3.1299e-01,  1.5117e+00, -5.3955e-01,  3.5376e-01,  7.1106e-02,\n",
       "                      -4.0454e-01,  1.5942e-01,  4.2163e-01,  3.6279e-01,  4.9866e-02,\n",
       "                      -4.3677e-01, -9.3811e-02, -3.4375e-01, -2.3450e-01, -1.8738e-01,\n",
       "                       3.4473e-01, -4.7778e-01, -6.8359e-01,  8.4863e-01,  6.0059e-02,\n",
       "                      -2.1411e-01,  3.1860e-01,  1.1778e-03,  1.1650e-02,  1.6787e+00,\n",
       "                      -6.5979e-02, -9.0515e-02, -3.1812e-01, -8.8770e-01, -1.5820e-01,\n",
       "                       7.1899e-02,  3.5107e-01, -1.2964e-01, -2.3849e-02,  5.9229e-01,\n",
       "                       2.5195e-01,  1.1377e-01, -6.7725e-01,  1.6495e-02, -1.1127e-01,\n",
       "                      -1.8860e-01,  1.0474e-01,  6.4209e-01,  2.0117e-01,  2.6611e-01,\n",
       "                       1.2915e-01, -1.8738e-01, -2.8687e-01,  3.3228e-01,  1.1786e-01,\n",
       "                      -4.0283e-01,  1.1133e+00,  5.0928e-01,  3.9453e-01,  4.0576e-01,\n",
       "                      -2.1074e+00, -6.3037e-01,  1.2617e+00,  6.9678e-01,  2.0044e-01,\n",
       "                       2.7051e-01,  2.8369e-01, -1.1035e+00, -3.3862e-01,  8.4668e-01,\n",
       "                      -3.1465e+00,  6.9971e-01, -3.6206e-01,  5.8936e-01,  1.9238e+00,\n",
       "                      -3.4106e-01, -6.3672e-01,  5.8838e-01,  6.5979e-02,  1.2384e-01,\n",
       "                      -9.5215e-03,  4.0308e-01, -1.0901e-01,  9.8047e-01,  4.6844e-02,\n",
       "                       5.7275e-01,  1.3242e+00, -1.2871e+00,  3.3667e-01, -1.9580e-01,\n",
       "                      -6.9678e-01,  4.8859e-02,  4.9048e-01, -4.0859e+00,  6.3525e-01,\n",
       "                      -2.2498e-01,  4.0112e-01,  1.6543e+00,  8.9453e-01, -4.0332e-01,\n",
       "                      -6.4502e-01,  4.9390e-01,  6.2598e-01, -1.1953e+00, -1.3708e-01,\n",
       "                      -5.4443e-01,  5.3125e-01, -2.2791e-01, -4.7754e-01,  3.4937e-01,\n",
       "                      -6.2109e-01, -6.0303e-01, -1.3271e+00,  2.0569e-01, -5.3271e-01,\n",
       "                      -9.5264e-01, -5.7715e-01, -7.7539e-01, -3.0859e-01, -1.7285e-01,\n",
       "                      -1.1934e+00,  1.7041e-01,  3.1421e-01,  8.1104e-01, -1.8265e-02,\n",
       "                       1.0162e-01,  1.6499e-03, -2.9648e-02, -3.9978e-02,  2.3636e-02,\n",
       "                       7.0862e-02, -1.0333e-01, -1.7685e-02, -9.2850e-03, -1.1536e-02,\n",
       "                       2.9617e-02,  1.3824e-02,  1.5610e-02,  1.0292e-02,  2.6779e-02,\n",
       "                       3.3569e-02, -7.9956e-03,  1.0193e-01,  1.2672e-02, -8.6548e-02,\n",
       "                      -1.6296e-02, -1.5979e-01, -5.3040e-02, -7.6447e-03,  7.9468e-02,\n",
       "                      -1.0065e-01,  2.7359e-02,  5.3741e-02,  9.4299e-03,  6.6467e-02,\n",
       "                       2.0386e-02, -8.1116e-02,  1.7271e-03,  1.0643e-03, -7.2510e-02,\n",
       "                       7.1533e-02, -1.2993e-02,  4.6417e-02, -2.0355e-02, -2.0905e-02,\n",
       "                       9.7580e-03, -2.8381e-02,  2.7023e-02,  1.9485e-02,  8.6121e-02,\n",
       "                      -3.8662e-03,  5.9814e-03, -4.0131e-02,  3.6804e-02, -1.0674e-02,\n",
       "                      -2.3209e-02, -1.3519e-02,  5.8060e-03, -8.6060e-03,  3.1681e-03,\n",
       "                      -7.9498e-03, -2.8030e-02, -6.4331e-02,  6.2683e-02, -2.2797e-02,\n",
       "                       1.9302e-02,  4.1656e-02,  2.1988e-02])),\n",
       "             ('transformer.h.4.self_attention.dense.weight',\n",
       "              tensor([[ 2.4052e-03, -1.6113e-02,  5.6343e-03,  ...,  8.1396e-04,\n",
       "                        1.5808e-02, -2.3460e-03],\n",
       "                      [-7.1487e-03, -1.4114e-03,  1.4755e-02,  ..., -1.7380e-02,\n",
       "                        1.1501e-03,  2.0737e-02],\n",
       "                      [-8.0414e-03, -3.0079e-03, -1.1375e-02,  ..., -8.2321e-03,\n",
       "                        4.7646e-03,  1.4114e-03],\n",
       "                      ...,\n",
       "                      [ 3.0289e-02, -1.4439e-03, -2.5040e-02,  ..., -2.3365e-03,\n",
       "                        4.0512e-03, -2.1637e-02],\n",
       "                      [ 5.6458e-03, -3.7014e-05, -6.0043e-03,  ...,  8.9569e-03,\n",
       "                        1.6098e-02,  3.8605e-02],\n",
       "                      [ 4.8709e-04,  1.7138e-03, -3.8242e-03,  ...,  5.7526e-03,\n",
       "                       -3.8166e-03, -4.8208e-04]])),\n",
       "             ('transformer.h.4.self_attention.dense.bias',\n",
       "              tensor([ 0.0176, -0.0156,  0.0053,  ..., -0.0704, -0.0489, -0.0146])),\n",
       "             ('transformer.h.4.post_attention_layernorm.weight',\n",
       "              tensor([1.1172, 1.3301, 1.0098,  ..., 1.5938, 1.4160, 0.7451])),\n",
       "             ('transformer.h.4.post_attention_layernorm.bias',\n",
       "              tensor([ 0.2255, -0.1992, -0.0338,  ..., -0.0398, -0.1149, -0.1043])),\n",
       "             ('transformer.h.4.mlp.dense_h_to_4h.weight',\n",
       "              tensor([[-0.0008,  0.0206,  0.0160,  ...,  0.0027,  0.0075, -0.0186],\n",
       "                      [-0.0206, -0.0131, -0.0017,  ...,  0.0471,  0.0144,  0.0226],\n",
       "                      [-0.0076,  0.0050,  0.0267,  ..., -0.0022,  0.0034, -0.0164],\n",
       "                      ...,\n",
       "                      [-0.0235,  0.0379, -0.0104,  ...,  0.0359,  0.0090,  0.0197],\n",
       "                      [-0.0049,  0.0284,  0.0167,  ..., -0.0148,  0.0032,  0.0444],\n",
       "                      [-0.0435,  0.0200, -0.0136,  ..., -0.0041,  0.0036,  0.0860]])),\n",
       "             ('transformer.h.4.mlp.dense_h_to_4h.bias',\n",
       "              tensor([-9.5215e-02, -4.7656e-01, -4.2748e-04,  ..., -4.1064e-01,\n",
       "                      -1.3062e-01, -1.3458e-02])),\n",
       "             ('transformer.h.4.mlp.dense_4h_to_h.weight',\n",
       "              tensor([[-5.5847e-03,  5.5695e-03, -6.1951e-03,  ..., -8.3847e-03,\n",
       "                        2.6932e-03,  2.7725e-02],\n",
       "                      [-9.9869e-03, -7.3166e-03, -8.3113e-04,  ..., -1.0483e-02,\n",
       "                       -1.8555e-02, -8.4686e-03],\n",
       "                      [-2.4719e-03, -2.9617e-02, -2.4750e-02,  ...,  4.6897e-04,\n",
       "                       -2.5696e-02,  1.3981e-03],\n",
       "                      ...,\n",
       "                      [ 2.9397e-04,  1.9028e-02, -3.9444e-03,  ...,  6.1378e-03,\n",
       "                        2.7835e-05,  7.6561e-03],\n",
       "                      [ 3.4466e-03, -1.6510e-02,  1.3000e-02,  ..., -7.5645e-03,\n",
       "                       -4.0016e-03, -2.1683e-02],\n",
       "                      [-8.8882e-03,  2.5467e-02,  5.2910e-03,  ...,  1.4580e-02,\n",
       "                       -3.1090e-04, -6.9946e-02]])),\n",
       "             ('transformer.h.4.mlp.dense_4h_to_h.bias',\n",
       "              tensor([ 0.0264,  0.0022,  0.0329,  ...,  0.0106,  0.0147, -0.0184])),\n",
       "             ('transformer.h.5.input_layernorm.weight',\n",
       "              tensor([0.9976, 1.3750, 0.9834,  ..., 1.5879, 1.4814, 0.7632])),\n",
       "             ('transformer.h.5.input_layernorm.bias',\n",
       "              tensor([-0.0587,  0.0269,  0.0801,  ...,  0.0014, -0.0745, -0.0084])),\n",
       "             ('transformer.h.5.self_attention.query_key_value.weight',\n",
       "              tensor([[ 0.0057, -0.0006,  0.0036,  ..., -0.0003, -0.0058, -0.0297],\n",
       "                      [ 0.0120, -0.0002, -0.0005,  ...,  0.0056,  0.0085, -0.0005],\n",
       "                      [-0.0105,  0.0126,  0.0310,  ...,  0.0077, -0.0012,  0.0017],\n",
       "                      ...,\n",
       "                      [ 0.0214, -0.0007,  0.0061,  ...,  0.0331, -0.0208, -0.0072],\n",
       "                      [-0.0014,  0.0377,  0.0081,  ...,  0.0062,  0.0155, -0.0264],\n",
       "                      [ 0.0277, -0.0010,  0.0053,  ..., -0.0087, -0.0258,  0.0014]])),\n",
       "             ('transformer.h.5.self_attention.query_key_value.bias',\n",
       "              tensor([ 1.8982e-02,  2.1057e-02,  2.2354e-02, -9.8145e-02,  7.6904e-02,\n",
       "                      -5.5176e-02, -2.2797e-02,  4.9805e-02, -7.8506e-03,  1.7929e-02,\n",
       "                      -2.7481e-02, -3.3875e-02, -3.4454e-02, -9.0485e-03,  4.8187e-02,\n",
       "                       1.1804e-01,  3.5278e-02,  5.0720e-02, -4.5853e-03,  4.9515e-03,\n",
       "                       5.5939e-02, -6.3232e-02, -6.0913e-02,  7.7576e-02, -3.4912e-02,\n",
       "                      -7.4219e-02, -3.5889e-02,  6.2744e-02,  7.9422e-03, -6.7505e-02,\n",
       "                      -1.4626e-02, -1.3838e-03,  4.2572e-02, -9.4833e-03,  1.9257e-02,\n",
       "                       1.6434e-02,  5.5504e-03, -4.8141e-03,  6.2134e-02,  7.3700e-03,\n",
       "                       6.0516e-02,  1.1025e+00, -1.2183e-01, -6.4697e-03,  5.7556e-02,\n",
       "                      -2.6016e-02,  2.8183e-02, -1.8570e-02, -6.3324e-03,  1.3565e-02,\n",
       "                      -1.2646e-01, -5.2376e-03, -1.2512e-02, -1.6998e-02,  5.4413e-02,\n",
       "                       1.2341e-03, -5.6922e-05, -3.8361e-02,  1.9485e-02, -8.4717e-02,\n",
       "                       9.7656e-02, -4.5532e-02, -4.0802e-02,  9.2926e-03,  1.0303e-01,\n",
       "                       1.4001e-01, -1.5393e-01, -3.3521e-01,  1.4124e-01,  1.9458e-01,\n",
       "                      -1.3440e-01,  1.6687e-01,  6.6357e-01,  3.9368e-02,  5.6427e-02,\n",
       "                       4.4739e-02, -2.7466e-01, -1.4355e-01, -2.7808e-01, -1.1688e-01,\n",
       "                      -2.1680e-01,  7.0923e-02, -2.2656e-01, -9.7717e-02, -3.6621e-01,\n",
       "                      -7.8552e-02,  9.3811e-02,  2.1045e-01, -2.1692e-01, -9.5398e-02,\n",
       "                      -2.5122e-01,  2.0020e-02,  8.5693e-02,  1.2854e-01, -2.3376e-01,\n",
       "                      -2.9297e-01,  6.4819e-02, -7.1533e-02, -9.4531e-01, -2.1924e-01,\n",
       "                       5.0751e-02, -3.4607e-02,  2.3145e-01, -2.1088e-02, -1.6833e-01,\n",
       "                      -5.4717e-05, -2.0227e-01, -2.4170e-01, -7.0801e-02,  4.2627e-01,\n",
       "                      -2.6291e-02, -2.2705e-02,  1.6199e-01,  2.3086e+00, -4.6021e-02,\n",
       "                      -5.2881e-01,  7.0374e-02,  6.0303e-02, -3.9825e-02,  1.9775e-01,\n",
       "                      -1.6190e-02,  1.0400e-01, -7.2021e-02, -1.2769e-01,  2.6196e-01,\n",
       "                      -8.2397e-02,  2.1973e-01, -1.5979e-01,  6.1913e-03, -3.6896e-02,\n",
       "                       9.4376e-03,  2.8656e-02, -9.6741e-03,  2.1255e-02, -7.2083e-02,\n",
       "                      -7.5012e-02,  8.7891e-02,  5.4321e-03,  2.9434e-02, -6.0944e-02,\n",
       "                       3.7842e-02,  1.2772e-02, -3.7842e-02,  3.2349e-02,  2.0309e-02,\n",
       "                      -3.7140e-02, -1.2749e-02,  1.5976e-02, -4.5624e-02,  2.3285e-02,\n",
       "                      -3.9795e-02, -2.5391e-02,  2.1713e-02, -2.7054e-02,  6.5727e-03,\n",
       "                       4.3304e-02,  3.8666e-02,  4.2297e-02,  6.4697e-02, -1.0162e-02,\n",
       "                       2.5345e-02,  4.9438e-02, -2.0218e-02, -7.4272e-03,  1.4725e-02,\n",
       "                       2.4490e-02, -5.0659e-02, -5.3314e-02, -7.6416e-02,  1.9470e-02,\n",
       "                       1.7868e-02, -5.3070e-02, -1.6876e-02, -1.6663e-02, -2.2964e-02,\n",
       "                      -4.7180e-02,  1.2756e-02, -3.3386e-02,  5.3711e-03, -1.2161e-02,\n",
       "                       7.2956e-04,  1.1359e-01,  2.0046e-03, -4.7089e-02,  2.1866e-02,\n",
       "                       1.3863e-02, -5.0720e-02, -1.0358e-01, -5.9395e-03,  3.3417e-02,\n",
       "                      -3.3997e-02,  3.6041e-02, -1.5674e-01,  1.8042e-01, -1.4185e-01,\n",
       "                      -3.4326e-01,  2.0496e-01,  1.0321e-01,  2.9883e-01,  3.0957e-01,\n",
       "                       3.4277e-01, -1.3330e-01, -1.4319e-01,  7.7637e-02,  2.1106e-01,\n",
       "                      -3.5938e-01,  1.6919e-01,  1.5393e-01,  2.0820e+00,  5.3223e-02,\n",
       "                       3.9038e-01,  3.4106e-01, -2.2253e-01,  2.1619e-01, -2.3303e-01,\n",
       "                       4.2694e-02, -1.9617e-01,  1.0919e-01,  4.9438e-01,  1.8823e-01,\n",
       "                      -4.1016e-01,  2.3657e-01,  3.4741e-01,  1.4189e+00, -3.0908e-01,\n",
       "                      -2.9761e-01,  2.1057e-02, -7.9041e-02,  9.4223e-03, -4.4060e-04,\n",
       "                       3.0737e-01,  1.4478e-01, -2.8223e-01, -9.8022e-02,  1.1639e-01,\n",
       "                       4.2896e-01, -2.6855e-01,  6.2073e-02,  9.0515e-02,  7.2412e-01,\n",
       "                       1.8994e-01,  2.0654e-01, -4.7339e-01,  3.4119e-02, -6.0150e-02,\n",
       "                      -4.0894e-01, -1.8042e-01,  7.8247e-02,  2.2498e-01,  1.9324e-01,\n",
       "                       6.5491e-02,  3.9502e-01,  1.3330e-01, -2.0020e+00, -3.2368e-03,\n",
       "                       2.1497e-01,  1.7712e-01,  3.2007e-01,  6.2622e-02, -2.6172e-01,\n",
       "                      -8.8867e-02,  1.0583e-01, -4.4482e-01,  1.1884e-01,  5.8154e-01,\n",
       "                       1.7773e-01, -8.0795e-03,  3.6957e-02,  3.2593e-01, -2.4658e-02,\n",
       "                       3.6230e-01,  1.0577e-01,  1.3184e-01, -5.4834e-01,  5.4260e-02,\n",
       "                       1.8970e-01,  3.3203e-02,  8.0139e-02, -8.2092e-02, -2.4280e-01,\n",
       "                       1.7334e-02, -8.3313e-02, -1.7480e-01,  2.0520e-01, -1.1407e-01,\n",
       "                       3.3081e-01,  3.0664e-01,  1.4124e-01, -8.2092e-02, -2.1836e+00,\n",
       "                       1.4050e-01, -2.1521e-01, -2.3401e-01,  2.0676e-02, -9.6558e-02,\n",
       "                       1.7761e-01,  1.5417e-01, -1.8860e-01, -6.4888e-03, -7.3547e-02,\n",
       "                      -4.2542e-02,  2.4585e-01,  2.3755e-01,  2.4536e-01, -8.9600e-01,\n",
       "                       3.7256e-01, -1.7712e-01, -2.0584e-02,  1.9971e-01,  1.8274e-01,\n",
       "                       1.4001e-01,  3.5522e-01, -3.8135e-01, -1.4917e-01, -1.2097e-01,\n",
       "                       5.2734e-02, -3.1543e-01, -1.9897e-01, -4.0207e-03, -2.6294e-01,\n",
       "                      -1.0132e-01, -3.3374e-01,  1.3879e-01,  1.4319e-01,  9.0027e-02,\n",
       "                       4.6631e-02,  4.6753e-01, -7.2815e-02, -1.1780e-01,  1.0022e-01,\n",
       "                       1.6284e-01,  2.2205e-01,  5.0568e-02, -2.6001e-01,  1.2939e-01,\n",
       "                      -1.4734e-01,  4.6094e-01, -6.6406e-02, -3.9624e-01,  7.8979e-02,\n",
       "                       2.8467e-01, -2.9007e-02,  7.8796e-02, -1.7432e-01, -2.4402e-01,\n",
       "                       7.6270e-01,  8.9966e-02,  1.3939e-02,  1.1261e-01,  6.8481e-02,\n",
       "                       6.8359e-02,  1.3076e+00, -4.6387e-01, -9.2041e-02, -1.9482e-01,\n",
       "                      -7.1472e-02,  1.3672e-02,  9.6863e-02,  8.5297e-03, -1.2589e-02,\n",
       "                       1.7456e-01,  4.0820e-01,  1.0967e-05, -3.4888e-01, -2.2351e-01,\n",
       "                       3.3276e-01, -1.5405e-01,  7.9688e-01, -1.2634e-01,  1.9446e-01,\n",
       "                      -2.7313e-03, -2.3364e-01, -1.0828e-01, -3.1769e-02, -3.4961e-01,\n",
       "                       1.1102e-01,  2.7295e-01,  7.3669e-02,  5.0000e-01,  7.3975e-02,\n",
       "                       7.0801e-02, -2.4365e-01, -1.5173e-03,  1.5723e-01,  2.3743e-01,\n",
       "                      -1.0791e-01, -1.1176e-01, -3.0664e-01, -1.9299e-01, -2.1484e-01,\n",
       "                       3.0493e-01,  4.4897e-01, -1.0541e-01,  9.8190e-03, -2.6929e-01,\n",
       "                       1.3049e-01, -3.5840e-01, -1.2292e-01, -1.7725e-01, -3.8738e-03,\n",
       "                       2.9510e-02, -4.4287e-01, -8.1836e-01, -1.6333e-01,  7.5098e-01,\n",
       "                      -4.0967e-01,  3.2690e-01,  4.4336e-01, -5.0635e-01,  6.1377e-01,\n",
       "                      -2.0410e+00,  1.8213e-01,  1.8311e-01,  2.3352e-01,  2.2507e-02,\n",
       "                       1.5820e-01,  3.7556e-03, -4.0344e-02, -2.7026e-01, -2.3389e-01,\n",
       "                      -4.8193e-01,  2.8296e-01, -2.9077e-01,  3.1494e-01, -2.0801e-01,\n",
       "                      -2.1655e-01, -3.4637e-02, -6.4819e-02,  8.7463e-02, -3.1763e-01,\n",
       "                      -1.5808e-01, -1.2720e-01,  2.7451e-02,  1.5991e-01, -2.2375e-01,\n",
       "                      -2.5757e-01, -1.5906e-01,  4.9829e-01,  9.4971e-01, -1.7126e-01,\n",
       "                      -8.4839e-03, -7.7759e-02, -4.3164e-01, -1.8835e-01, -6.5308e-02,\n",
       "                       1.4099e-02, -4.0869e-01,  3.0396e-01, -3.2153e-01, -2.5146e-01,\n",
       "                      -1.2219e-01, -3.5425e-01, -1.3135e-01, -1.0162e-01, -2.7893e-02,\n",
       "                       3.9749e-03,  3.4302e-01,  3.2202e-01,  3.2275e-01,  6.4514e-02,\n",
       "                       1.1658e-01,  3.8574e-01,  2.4402e-01,  3.1769e-02, -1.1670e-01,\n",
       "                       4.6265e-01,  9.7900e-01,  6.5430e-02,  1.0059e-01,  5.6641e-01,\n",
       "                      -8.5205e-02, -2.1387e-01,  2.6367e-01, -4.0625e-01,  1.6553e+00,\n",
       "                      -7.4219e-02, -3.4155e-01, -4.5923e-01,  1.1127e-01, -1.2524e-01,\n",
       "                      -1.4294e-01, -2.3364e-01,  4.2749e-01, -2.8641e-02, -2.6050e-01,\n",
       "                       4.6326e-02, -8.4106e-02,  1.5625e-01,  2.0044e-01,  8.8440e-02,\n",
       "                      -8.6853e-02,  3.8605e-02,  1.8848e-01,  2.0715e-01,  8.0566e-02,\n",
       "                      -2.6398e-02, -2.3560e-01, -1.1255e-01, -2.6993e-02,  4.9561e-01,\n",
       "                       1.7310e-01, -1.2878e-01,  1.1263e-03,  2.1021e-01,  1.6943e-01,\n",
       "                       2.9419e-01,  7.3425e-02, -5.2887e-02,  2.6685e-01, -4.4238e-01,\n",
       "                       2.5970e-02,  1.4856e-01, -3.1616e-02,  8.7744e-01, -4.9243e-01,\n",
       "                      -2.6123e-01,  8.8184e-01, -2.4963e-01, -5.6006e-01, -3.8989e-01,\n",
       "                       2.3657e-01,  1.2080e+00, -6.0693e-01, -2.3865e-02, -3.6499e-01,\n",
       "                       4.7681e-01, -6.5381e-01,  1.0370e-01, -1.2314e-02,  1.9800e-01,\n",
       "                      -4.3799e-01, -5.2441e-01, -1.0187e-01, -4.1412e-02,  1.4270e-01,\n",
       "                      -2.1960e-01,  1.0625e+00, -2.0947e-01,  3.2861e-01, -8.1152e-01,\n",
       "                      -2.1692e-01,  5.8203e-01,  2.1826e-01,  4.3823e-02, -3.7085e-01,\n",
       "                      -6.6113e-01, -1.7510e+00, -3.7500e-01,  8.7256e-01,  5.0476e-02,\n",
       "                      -5.7764e-01, -6.7480e-01,  1.6174e-01,  3.8696e-01,  8.1787e-02,\n",
       "                       1.0246e-02,  2.1289e-01, -1.0732e+00, -6.8750e-01, -2.3535e-01,\n",
       "                       1.9485e-02,  1.2129e+00,  1.4246e-01, -4.6704e-01, -9.7217e-01,\n",
       "                       9.1858e-02,  1.1066e-01,  5.3131e-02,  8.3154e-01,  5.9180e-01,\n",
       "                      -1.1853e-01, -6.3770e-01, -3.0054e-01,  6.4697e-01, -2.3633e-01,\n",
       "                      -8.6084e-01,  8.8120e-03, -2.4490e-02,  4.9858e-03, -1.4858e-03,\n",
       "                       9.2468e-03, -6.5674e-02, -3.3630e-02, -2.0004e-02, -5.7945e-03,\n",
       "                      -1.3802e-02, -1.5701e-02, -1.4648e-02, -1.2863e-02,  4.8485e-03,\n",
       "                      -2.2049e-02,  9.3536e-03, -1.5881e-01, -2.9736e-03, -2.4475e-02,\n",
       "                      -2.1637e-02, -1.4320e-02, -4.6272e-03,  1.5312e-02, -1.4086e-03,\n",
       "                      -3.7201e-02, -1.5612e-03,  2.5253e-03,  2.8610e-02,  4.2786e-02,\n",
       "                      -3.3173e-02, -3.7415e-02,  1.4442e-02, -1.1772e-02,  4.6539e-03,\n",
       "                      -1.2718e-02,  1.2756e-02,  7.2823e-03, -1.0422e-02,  9.8779e-01,\n",
       "                       3.9337e-02, -3.2318e-02, -2.5879e-02,  1.0277e-02, -3.1982e-02,\n",
       "                       1.2245e-02, -2.7588e-02,  8.2715e-01,  3.8666e-02,  2.7027e-03,\n",
       "                      -4.7302e-03, -2.0905e-02, -9.9945e-03,  3.7506e-02, -1.9836e-04,\n",
       "                       8.4076e-03,  6.1264e-03,  3.0167e-02, -1.3618e-02, -9.1171e-03,\n",
       "                      -7.6180e-03, -3.0151e-02,  2.8824e-02,  2.0462e-02, -5.6229e-03,\n",
       "                      -2.4390e-01,  1.4478e-01, -9.1125e-02, -2.2046e-01,  2.1924e-01,\n",
       "                       3.7109e-01,  5.7526e-02, -6.0889e-01,  7.3059e-02,  8.6365e-03,\n",
       "                      -3.1567e-01,  9.6875e-01, -3.6816e-01,  3.5352e-01,  1.0986e+00,\n",
       "                      -1.3464e-01, -2.5146e-01,  3.1592e-01, -1.5991e-01,  6.0547e-01,\n",
       "                       2.0410e+00, -4.5264e-01, -2.4536e-01,  5.0684e-01,  6.2012e-01,\n",
       "                      -3.1567e-01,  3.8965e-01, -7.1777e-01, -6.5979e-02, -3.4521e-01,\n",
       "                      -6.7871e-01, -9.4336e-01, -2.2827e-01, -2.3779e-01,  7.6270e-01,\n",
       "                       1.3398e+00, -1.2549e-01, -4.6043e-03, -6.1401e-02, -2.4194e-01,\n",
       "                       8.6365e-03, -4.7827e-01, -2.8174e-01,  2.9883e-01,  5.6006e-01,\n",
       "                      -5.7520e-01, -1.4539e-01,  3.2642e-01,  6.8896e-01,  3.3203e-01,\n",
       "                      -3.2949e+00,  7.4524e-02, -6.1670e-01, -3.2764e-01, -6.5234e-01,\n",
       "                       2.0813e-01,  3.2568e-01,  6.4758e-02,  2.4023e-01,  4.0527e-01,\n",
       "                       2.1924e-01,  1.8384e-01, -2.4683e-01, -1.8518e-01,  2.4048e-02,\n",
       "                       3.3630e-02,  8.6823e-03, -8.6288e-03, -2.2869e-03,  1.7151e-02,\n",
       "                       5.5389e-02,  6.2103e-02, -4.2908e-02,  3.9581e-02,  3.1494e-02,\n",
       "                      -5.7190e-02,  4.2725e-02,  3.4546e-02, -2.4551e-02,  2.1057e-03,\n",
       "                      -2.3087e-02, -8.0994e-02,  3.3894e-03, -4.6875e-02,  2.7405e-02,\n",
       "                      -4.9591e-02,  1.6050e-03, -3.5767e-02, -3.4821e-02, -4.0771e-02,\n",
       "                      -8.8196e-03,  2.6688e-02, -2.0264e-02, -3.4302e-02, -3.6804e-02,\n",
       "                       1.4038e-02,  3.4180e-02,  4.9782e-03, -5.9723e-02, -2.1271e-02,\n",
       "                       3.4485e-02,  1.6998e-02,  1.3618e-02,  4.9734e-04, -2.0721e-02,\n",
       "                      -5.3772e-02, -2.3239e-02,  8.5297e-03, -6.7759e-04, -6.4331e-02,\n",
       "                       1.6623e-03,  1.9928e-02, -1.3138e-02,  7.8440e-04, -2.1851e-02,\n",
       "                      -3.3356e-02, -1.1452e-02,  1.2070e-02,  2.3056e-02,  6.0806e-03,\n",
       "                       6.5063e-02, -3.6194e-02, -3.1403e-02, -1.8816e-03, -4.8340e-02,\n",
       "                      -3.3903e-04,  7.1983e-03,  4.6997e-03])),\n",
       "             ('transformer.h.5.self_attention.dense.weight',\n",
       "              tensor([[-0.0049,  0.0091,  0.0079,  ..., -0.0083, -0.0132, -0.0230],\n",
       "                      [ 0.0003, -0.0021,  0.0166,  ..., -0.0059, -0.0353,  0.0132],\n",
       "                      [ 0.0119, -0.0105, -0.0053,  ...,  0.0047, -0.0210, -0.0050],\n",
       "                      ...,\n",
       "                      [ 0.0239,  0.0181,  0.0110,  ..., -0.0331, -0.0015,  0.0022],\n",
       "                      [ 0.0250,  0.0207, -0.0078,  ...,  0.0167, -0.0074,  0.0268],\n",
       "                      [-0.0003, -0.0043, -0.0033,  ..., -0.0020,  0.0105, -0.0043]])),\n",
       "             ('transformer.h.5.self_attention.dense.bias',\n",
       "              tensor([-0.0006, -0.0495,  0.0392,  ..., -0.0946, -0.0874, -0.0219])),\n",
       "             ('transformer.h.5.post_attention_layernorm.weight',\n",
       "              tensor([1.0703, 1.2246, 0.9448,  ..., 1.5605, 1.4043, 0.6860])),\n",
       "             ('transformer.h.5.post_attention_layernorm.bias',\n",
       "              tensor([ 0.1370,  0.0286,  0.0679,  ...,  0.1661, -0.0338, -0.0862])),\n",
       "             ('transformer.h.5.mlp.dense_h_to_4h.weight',\n",
       "              tensor([[ 0.0011, -0.0211, -0.0008,  ..., -0.0280,  0.0006,  0.0111],\n",
       "                      [ 0.0005, -0.0075,  0.0146,  ..., -0.0093, -0.0059,  0.0014],\n",
       "                      [-0.0090,  0.0587, -0.0068,  ...,  0.0036,  0.0296, -0.0221],\n",
       "                      ...,\n",
       "                      [-0.0124,  0.0294,  0.0070,  ...,  0.0014,  0.0142, -0.0158],\n",
       "                      [-0.0149,  0.0162,  0.0114,  ..., -0.0169,  0.0032, -0.0098],\n",
       "                      [-0.0029,  0.0076,  0.0027,  ...,  0.0248, -0.0064,  0.0171]])),\n",
       "             ('transformer.h.5.mlp.dense_h_to_4h.bias',\n",
       "              tensor([-0.2233, -0.1410, -0.6333,  ..., -0.3586, -0.2178, -0.0365])),\n",
       "             ('transformer.h.5.mlp.dense_4h_to_h.weight',\n",
       "              tensor([[-0.0088,  0.0044, -0.0009,  ..., -0.0117, -0.0011,  0.0178],\n",
       "                      [-0.0118,  0.0042,  0.0210,  ..., -0.0112, -0.0254, -0.0133],\n",
       "                      [ 0.0009, -0.0100, -0.0014,  ...,  0.0039, -0.0296,  0.0009],\n",
       "                      ...,\n",
       "                      [ 0.0074, -0.0221,  0.0043,  ..., -0.0089, -0.0011, -0.0167],\n",
       "                      [-0.0102,  0.0442, -0.0124,  ..., -0.0003,  0.0051, -0.0190],\n",
       "                      [-0.0112, -0.0008, -0.0146,  ..., -0.0102, -0.0062,  0.0032]])),\n",
       "             ('transformer.h.5.mlp.dense_4h_to_h.bias',\n",
       "              tensor([ 0.0355,  0.0585, -0.0451,  ..., -0.1589,  0.0267,  0.0027])),\n",
       "             ('transformer.h.6.input_layernorm.weight',\n",
       "              tensor([1.0156, 1.2979, 0.9917,  ..., 1.6416, 1.4365, 0.8589])),\n",
       "             ('transformer.h.6.input_layernorm.bias',\n",
       "              tensor([-0.0210,  0.0510,  0.1119,  ...,  0.0314, -0.0434,  0.0087])),\n",
       "             ('transformer.h.6.self_attention.query_key_value.weight',\n",
       "              tensor([[-1.1681e-02,  1.4900e-02, -1.1124e-02,  ..., -1.5839e-02,\n",
       "                       -8.3389e-03,  2.2736e-03],\n",
       "                      [ 8.7547e-04, -1.4236e-02, -5.6267e-03,  ...,  3.1799e-02,\n",
       "                       -1.1284e-02, -1.4992e-03],\n",
       "                      [-7.3776e-03, -3.5210e-03,  6.2065e-03,  ...,  2.2476e-02,\n",
       "                       -3.3997e-02, -2.9068e-03],\n",
       "                      ...,\n",
       "                      [ 4.0985e-02, -2.0447e-03,  2.4567e-02,  ..., -2.4078e-02,\n",
       "                       -1.9806e-02, -2.4548e-03],\n",
       "                      [ 4.4975e-03,  4.4479e-03,  1.0811e-02,  ...,  6.9458e-02,\n",
       "                       -5.2216e-02, -7.6141e-03],\n",
       "                      [ 3.1173e-05, -3.6987e-02, -1.2718e-02,  ..., -8.6451e-04,\n",
       "                       -5.4077e-02, -9.9659e-04]])),\n",
       "             ('transformer.h.6.self_attention.query_key_value.bias',\n",
       "              tensor([-1.9678e-01, -9.8572e-03,  2.4689e-02, -5.2368e-02, -4.5380e-02,\n",
       "                       2.2229e-01,  2.8122e-02, -7.6843e-02, -1.5747e-01,  8.8135e-02,\n",
       "                       2.1790e-02,  1.0870e-01,  1.4783e-01,  1.3757e-01, -1.6956e-01,\n",
       "                      -6.4209e-02, -7.9956e-02,  9.0332e-03,  1.0052e-01,  1.1816e-01,\n",
       "                       1.8115e-01, -2.0007e-01,  1.7664e-01, -1.7432e-01,  3.3252e-01,\n",
       "                      -1.5393e-01,  1.6101e-01, -1.6089e-01,  7.9163e-02,  1.3257e-01,\n",
       "                       1.1462e-01,  9.8022e-02,  6.1493e-02,  2.4109e-01, -1.0022e-01,\n",
       "                      -1.0730e-01,  9.3317e-04, -1.1407e-01,  1.7981e-01,  5.4291e-02,\n",
       "                      -9.9060e-02, -2.0593e-01,  1.8591e-01,  1.7767e-03,  9.8816e-02,\n",
       "                      -9.9182e-02, -6.5430e-02,  2.4622e-01, -1.2830e-01,  1.7371e-01,\n",
       "                      -5.6427e-02, -1.1314e-02,  7.9407e-02, -2.4646e-01, -4.8706e-02,\n",
       "                       5.3925e-02,  2.1655e-01,  8.3313e-02,  1.9409e-01,  7.4707e-02,\n",
       "                      -1.5063e-01,  8.4473e-02,  1.8201e-01, -2.9907e-02,  8.0444e-02,\n",
       "                       2.7832e-01,  8.7830e-02,  1.1896e-01, -1.8542e-01, -1.1620e-02,\n",
       "                      -1.5515e-01,  3.9429e-01, -1.3928e-01, -2.1582e+00, -2.3865e-01,\n",
       "                      -4.6436e-01,  1.9751e-01,  1.4746e-01, -1.5112e-01, -1.8628e-01,\n",
       "                       7.0068e-02,  1.3391e-01,  1.9855e-03, -1.7102e-01, -9.9243e-02,\n",
       "                       9.6252e-02, -4.5868e-02,  4.2700e-01,  4.2114e-02,  3.2883e-03,\n",
       "                       5.4138e-02, -2.5234e-03,  2.7295e-01, -3.1836e-01, -1.8585e-02,\n",
       "                      -1.2500e-01,  2.3804e-02, -6.5979e-02,  3.0899e-04,  1.1963e-01,\n",
       "                       9.1125e-02,  1.8774e-01, -6.4575e-02,  2.0764e-01,  1.1975e-01,\n",
       "                      -1.4368e-01,  2.4231e-02, -1.0689e-02,  1.9006e-01, -1.7627e-01,\n",
       "                       2.5558e-02,  1.7688e-01,  3.5034e-02,  5.0171e-02,  1.6724e-01,\n",
       "                       6.2164e-02,  9.3567e-02, -5.8022e-03, -2.2070e-01,  7.8308e-02,\n",
       "                       1.0699e-01, -6.9043e-01,  3.6438e-02,  1.5656e-02,  3.5889e-02,\n",
       "                      -1.1139e-01, -1.0840e-01, -9.4971e-02,  2.1362e-02,  9.8324e-04,\n",
       "                      -2.0508e-02, -6.5193e-03,  4.3549e-02,  1.4870e-02,  3.9581e-02,\n",
       "                       9.5520e-03,  1.5602e-02,  3.5718e-01,  1.0826e-02, -1.2421e-02,\n",
       "                       6.8512e-03, -4.1107e-02, -6.8054e-02,  4.0588e-03,  4.4022e-03,\n",
       "                      -9.6893e-03,  1.9043e-02, -1.7517e-02,  3.0334e-02, -1.9913e-02,\n",
       "                       3.8574e-02, -5.6244e-02, -1.7059e-02, -9.9564e-03,  1.9547e-02,\n",
       "                       2.9846e-02, -2.9016e-04,  7.2449e-02,  3.4637e-02, -4.3671e-02,\n",
       "                      -1.9791e-02,  7.3013e-03,  9.1858e-03, -8.2016e-03,  5.4688e-02,\n",
       "                       3.4973e-02,  3.9032e-02, -5.9296e-02, -5.1025e-02, -5.3406e-02,\n",
       "                       5.3711e-02, -2.9099e-02, -1.4931e-02, -7.0343e-03,  5.7709e-02,\n",
       "                       1.5366e-02, -4.2358e-02,  4.4556e-02,  5.9128e-03, -9.1431e-02,\n",
       "                      -3.4485e-02,  1.0971e-02,  3.4088e-02, -2.9083e-02,  2.2003e-02,\n",
       "                       8.4473e-02, -1.6983e-02, -3.7964e-02, -9.4299e-03, -1.8127e-02,\n",
       "                       3.3665e-04,  1.8814e-02, -3.0103e-01,  5.3162e-02,  3.9520e-02,\n",
       "                       1.0461e-01,  3.0005e-01,  2.2864e-01,  1.3574e-01, -6.5063e-02,\n",
       "                      -6.2842e-01,  6.7368e-03, -1.2231e-01,  3.4546e-02, -4.7974e-02,\n",
       "                      -3.1177e-01,  1.3290e-02, -1.3953e-01, -4.4556e-02, -2.5574e-02,\n",
       "                      -2.3413e-01, -4.4751e-01,  7.0007e-02,  9.5581e-02, -4.4556e-01,\n",
       "                       2.5732e-01, -6.7505e-02, -2.1934e+00,  8.2947e-02, -8.5107e-01,\n",
       "                      -4.2267e-02, -1.7773e-01,  4.3799e-01, -1.9055e-01,  2.1484e-01,\n",
       "                       1.2140e-01,  2.2791e-01,  2.5439e-01, -2.9907e-01,  9.9792e-02,\n",
       "                      -1.1986e-02, -7.2937e-02,  1.0541e-01,  5.2185e-02,  2.0020e-01,\n",
       "                      -1.4246e-01, -2.9932e-01, -1.2878e-01,  1.6162e-01, -3.9276e-02,\n",
       "                       1.6617e-02, -7.5684e-02,  2.1021e-01, -2.5830e-01, -5.7520e-01,\n",
       "                       6.7566e-02,  5.4413e-02,  2.1301e-01, -9.7168e-02,  8.9111e-02,\n",
       "                      -1.1963e-02, -3.7903e-02, -3.3997e-02,  3.1372e-01, -9.8816e-02,\n",
       "                       2.3535e-01,  2.3608e-01,  5.5969e-02,  3.4570e-01, -2.8296e-01,\n",
       "                       5.3320e-01,  8.4717e-01, -1.1816e+00,  7.5073e-02,  8.7500e-01,\n",
       "                      -9.6240e-01,  2.0957e+00,  5.0342e-01,  3.1177e-01,  2.1582e+00,\n",
       "                       1.5137e-01,  7.9346e-03,  1.1975e-01, -7.7344e-01, -8.7402e-01,\n",
       "                       1.1826e+00,  3.8843e-01,  6.5918e-01, -5.8441e-02, -9.3262e-01,\n",
       "                      -1.3428e-01,  1.4639e+00,  1.3008e+00, -3.6621e-01,  1.4473e+00,\n",
       "                       4.7913e-02,  3.4814e-01, -1.6807e+00, -5.8398e-01,  3.8428e-01,\n",
       "                       5.8105e-01,  1.1401e-01,  3.0078e-01,  1.0551e-02,  3.0258e-02,\n",
       "                       2.9678e-03, -6.1279e-01, -3.5474e-01, -2.1348e+00,  3.5474e-01,\n",
       "                       7.4658e-01, -1.2959e+00,  2.7734e-01, -1.1856e-02,  6.7505e-02,\n",
       "                      -1.8477e+00, -2.9468e-01,  1.9971e-01,  7.3547e-02,  3.0591e-01,\n",
       "                      -2.2229e-01,  2.0764e-01, -2.3474e-01,  2.1252e-01, -3.8281e-01,\n",
       "                       8.2861e-01, -2.0469e+00,  9.9976e-02, -1.9263e-01,  6.6309e-01,\n",
       "                      -8.4717e-01, -5.1953e-01, -4.4604e-01,  5.3125e-01, -9.8145e-01,\n",
       "                      -2.1914e+00,  2.2285e+00, -3.1104e-01, -2.0684e+00,  1.8906e+00,\n",
       "                      -4.9844e+00, -1.9824e+00, -3.2861e-01, -5.2109e+00,  4.8248e-02,\n",
       "                      -2.1570e-01, -2.6562e-01,  1.4238e+00,  2.1758e+00, -2.7402e+00,\n",
       "                      -8.4619e-01, -1.5449e+00, -2.2461e-01,  1.9932e+00,  4.5874e-01,\n",
       "                      -3.9355e+00, -2.8984e+00,  5.7129e-01, -2.7832e+00,  1.0803e-02,\n",
       "                      -5.1709e-01,  3.4609e+00,  1.0225e+00, -1.2969e+00, -1.4141e+00,\n",
       "                      -8.5266e-02, -7.5586e-01, -9.0408e-03, -2.0462e-02, -2.6782e-01,\n",
       "                       1.1963e+00,  9.0186e-01,  4.9609e+00, -5.4932e-01, -1.7695e+00,\n",
       "                       2.5645e+00, -7.8711e-01, -8.9722e-02, -5.0537e-01,  4.0000e+00,\n",
       "                      -2.2192e-01, -4.8608e-01, -1.0645e-01, -1.5469e+00,  1.9214e-01,\n",
       "                      -3.9868e-01,  2.9614e-01, -6.8945e-01,  1.1201e+00, -1.7549e+00,\n",
       "                       4.8750e+00,  4.3750e-01,  6.5283e-01, -1.0020e+00, -5.0087e-03,\n",
       "                      -3.5889e-02, -6.9962e-03,  2.6398e-02,  1.1406e-02, -1.3260e-02,\n",
       "                       2.2263e-02,  8.3694e-03, -6.6452e-03, -1.1711e-03, -1.7380e-02,\n",
       "                      -5.8651e-04,  8.8501e-03,  7.5340e-03, -5.3482e-03,  1.3840e-02,\n",
       "                       6.3133e-03, -4.1656e-03,  1.2238e-02, -2.0432e-02,  6.9656e-03,\n",
       "                       1.2222e-02,  1.4824e-02, -9.4604e-03,  4.3488e-03,  5.8708e-03,\n",
       "                      -8.8730e-03,  4.4281e-02, -6.1989e-03,  7.9727e-03,  1.2779e-02,\n",
       "                      -7.0114e-03, -2.0157e-02,  6.9695e-03, -1.3916e-02,  2.8656e-02,\n",
       "                      -7.5226e-03,  4.4656e-04, -1.7166e-02,  2.2720e-02,  1.4801e-02,\n",
       "                      -1.7548e-02,  2.8839e-02, -2.1420e-03,  7.7629e-03,  1.8387e-02,\n",
       "                       7.7782e-03,  1.4477e-03,  2.3071e-02, -2.4857e-02, -8.9493e-03,\n",
       "                       2.0584e-02,  1.1368e-02, -7.1297e-03,  1.2512e-02,  2.9278e-03,\n",
       "                       1.5518e-02, -1.0368e-02,  8.2932e-03,  2.2476e-02,  1.9928e-02,\n",
       "                      -6.0986e-01, -3.1006e-02,  8.0943e-05, -1.0767e-01, -2.6953e-01,\n",
       "                       1.2207e-01, -7.2607e-01, -1.3242e+00,  2.0508e-01, -5.3271e-01,\n",
       "                       1.9318e-02, -2.9238e+00,  8.6670e-01, -1.5857e-01,  7.5256e-02,\n",
       "                       8.0859e-01,  3.1372e-01,  4.6460e-01, -6.2012e-01,  3.7183e-01,\n",
       "                      -1.2568e+00, -1.2852e+00,  2.7197e-01, -6.2500e-01, -1.0382e-01,\n",
       "                       2.4277e+00, -1.5732e+00, -7.3792e-02,  2.3010e-01, -1.1711e-02,\n",
       "                       4.2871e-01,  4.2017e-01,  8.3691e-01,  4.4604e-01, -2.7939e-02,\n",
       "                       1.2140e-01, -5.2344e-01,  1.0059e+00, -1.3477e-01,  2.1953e+00,\n",
       "                      -3.2007e-01,  2.6211e+00, -8.1299e-01,  1.7188e-01, -6.2695e-01,\n",
       "                       4.0063e-01, -3.8501e-01, -1.4758e-01,  2.2607e-01,  3.3276e-01,\n",
       "                       3.5815e-01,  6.5039e-01, -6.8750e-01, -8.9844e-01, -4.5068e-01,\n",
       "                      -8.3496e-01,  3.4668e-01,  4.1211e-01, -7.3730e-02,  4.6411e-01,\n",
       "                      -2.6172e+00, -3.8745e-01, -4.8779e-01,  4.8901e-01,  1.6736e-01,\n",
       "                      -4.3628e-01, -4.9585e-01,  1.5762e+00,  2.6719e+00, -5.4492e-01,\n",
       "                      -3.6523e+00,  7.3608e-02, -5.7910e-01,  9.7363e-01,  1.4219e+00,\n",
       "                       3.6279e-01, -5.2227e+00, -5.8685e-02, -9.7461e-01,  1.4990e-01,\n",
       "                       2.5928e-01, -1.2754e+00,  5.4883e-01, -6.6914e+00,  5.9521e-01,\n",
       "                       1.3359e+00,  1.5088e+00,  5.3760e-01,  1.3828e+00,  1.4238e+00,\n",
       "                       1.3135e+00,  1.1387e+00, -4.6602e+00, -1.3193e+00,  6.1670e-01,\n",
       "                      -4.3237e-01, -5.9570e-01,  2.2681e-01,  1.6758e+00, -1.0596e-01,\n",
       "                       3.3057e-01, -4.7925e-01,  3.7891e-01,  2.7539e-01, -6.9482e-01,\n",
       "                       7.8467e-01,  5.6914e+00,  5.4541e-01, -4.5337e-01,  1.0020e+00,\n",
       "                      -9.7534e-02, -7.0361e-01,  1.0342e+00, -5.6836e-01,  6.5674e-02,\n",
       "                       4.5508e-01,  2.6660e-01,  1.0068e+00, -8.6865e-01, -6.6895e-01,\n",
       "                       4.2812e+00, -6.8408e-01,  5.4053e-01,  1.7480e+00,  7.7490e-01,\n",
       "                      -1.4233e-01, -4.1089e-01,  4.3828e+00, -1.3975e+00, -6.5479e-01,\n",
       "                      -5.3955e-01, -9.2850e-03,  4.8859e-02,  1.5011e-03,  2.6474e-02,\n",
       "                      -4.8096e-02,  6.8054e-03, -1.0000e+00, -3.3302e-03,  2.6306e-02,\n",
       "                       4.4785e-03,  2.5223e-02,  6.4270e-02,  5.8929e-02, -1.6159e-02,\n",
       "                      -4.3518e-02, -9.2087e-03, -6.1554e-02,  1.0941e-02, -3.0472e-02,\n",
       "                       2.1305e-03, -4.2053e-02, -2.9190e-02, -1.1894e-02,  1.9363e-02,\n",
       "                      -4.4250e-02,  1.3153e-02,  8.8318e-02, -2.8122e-02, -1.4442e-02,\n",
       "                       2.2110e-02, -1.5747e-02,  9.6802e-02, -3.0258e-02, -4.4006e-02,\n",
       "                      -5.7495e-02, -8.7814e-03, -1.5976e-02,  1.4526e-02,  2.5269e-02,\n",
       "                       7.1239e-04, -7.1001e-04,  3.2471e-02,  1.8663e-03,  1.0941e-02,\n",
       "                       2.5894e-02,  1.5221e-02, -5.3955e-02, -1.8988e-03,  4.6753e-02,\n",
       "                       6.8726e-02, -7.8201e-03,  3.1708e-02, -1.2192e-02,  4.7089e-02,\n",
       "                      -5.5878e-02, -1.9150e-02,  2.5085e-02,  1.5594e-02,  1.3954e-02,\n",
       "                       3.3569e-03, -6.0120e-02,  2.7283e-02,  8.9722e-02,  1.9958e-02,\n",
       "                       1.6418e-02,  3.5522e-01,  7.1411e-02, -6.7529e-01, -3.9893e-01,\n",
       "                       6.8994e-01,  2.4084e-01, -1.9531e-01,  9.7717e-02,  2.8711e-01,\n",
       "                      -2.8027e-01,  2.7539e-01, -9.1370e-02,  2.3254e-01, -2.2083e-01,\n",
       "                      -1.7324e+00, -2.1289e+00, -5.1709e-01,  5.5298e-02,  1.3623e-01,\n",
       "                      -2.7905e-01,  6.2561e-03,  1.2634e-01,  4.6802e-01,  8.4045e-02,\n",
       "                      -1.6748e-01, -2.3682e-01, -2.3535e-01,  1.3428e-01, -1.5979e-01,\n",
       "                       5.4688e-01,  3.2227e-01,  4.0869e-01, -1.6356e-03,  4.9121e-01,\n",
       "                      -3.4155e-01,  3.0737e-01, -2.1035e+00,  6.7627e-02, -9.6191e-02,\n",
       "                       2.1758e+00,  2.2656e-01, -3.4595e-01, -1.9453e+00, -3.3276e-01,\n",
       "                      -1.3443e-02, -1.4572e-02, -2.0813e-01, -2.7271e-01, -6.0400e-01,\n",
       "                      -3.3545e-01,  8.4863e-01,  2.3291e-01, -3.4332e-02, -1.4587e-01,\n",
       "                       3.8086e-01, -6.0120e-02, -3.2935e-01, -2.3096e-01, -3.1445e-01,\n",
       "                       1.8274e-01,  3.7915e-01, -3.4546e-01, -5.9131e-01, -1.4514e-01,\n",
       "                      -7.8201e-03, -1.5594e-02,  2.0172e-02,  8.3847e-03, -2.9205e-02,\n",
       "                      -4.1580e-03, -2.1652e-02,  2.9404e-02, -3.4065e-03, -4.8157e-02,\n",
       "                       3.4523e-03,  5.7316e-04, -1.3306e-02, -1.8143e-02,  2.8839e-02,\n",
       "                       3.7292e-02,  2.7122e-03,  2.0981e-02,  5.5504e-04, -2.9488e-03,\n",
       "                       5.8746e-03,  1.2222e-02,  3.6530e-02,  1.0323e-02,  2.6367e-02,\n",
       "                       1.0118e-03,  1.5793e-02, -1.0834e-02, -2.8641e-02,  4.2610e-03,\n",
       "                      -2.0584e-02,  1.5205e-02,  1.7868e-02, -4.3518e-02,  2.9099e-02,\n",
       "                       2.9800e-02,  1.2817e-02, -1.6312e-02,  5.3310e-04,  7.0724e-03,\n",
       "                      -1.5930e-02,  7.1335e-03, -1.0315e-02,  1.5152e-02,  4.8950e-02,\n",
       "                      -2.5940e-02,  5.4169e-02, -3.0792e-02, -1.8021e-02, -3.5889e-02,\n",
       "                      -6.3133e-03, -2.0959e-01,  1.5373e-03, -2.7878e-02, -6.2195e-02,\n",
       "                      -3.2745e-02,  2.5742e-02, -3.2406e-03,  6.9885e-03, -2.4002e-02,\n",
       "                       6.9656e-03,  1.7273e-02, -3.5034e-02])),\n",
       "             ('transformer.h.6.self_attention.dense.weight',\n",
       "              tensor([[-0.0055,  0.0098, -0.0047,  ...,  0.0498,  0.0008, -0.0050],\n",
       "                      [-0.0065, -0.0206, -0.0076,  ...,  0.0210, -0.0103, -0.0166],\n",
       "                      [ 0.0057, -0.0010, -0.0042,  ..., -0.0167,  0.0025,  0.0124],\n",
       "                      ...,\n",
       "                      [ 0.0066, -0.0123, -0.0434,  ..., -0.0041, -0.0008,  0.0196],\n",
       "                      [ 0.0189,  0.0153, -0.0139,  ...,  0.0067, -0.0231, -0.0202],\n",
       "                      [ 0.0080,  0.0017,  0.0078,  ..., -0.0001, -0.0071,  0.0166]])),\n",
       "             ('transformer.h.6.self_attention.dense.bias',\n",
       "              tensor([-0.0152,  0.0685,  0.0729,  ..., -0.0770, -0.0834, -0.0123])),\n",
       "             ('transformer.h.6.post_attention_layernorm.weight',\n",
       "              tensor([1.0410, 1.3477, 0.9453,  ..., 1.6855, 1.5098, 0.6318])),\n",
       "             ('transformer.h.6.post_attention_layernorm.bias',\n",
       "              tensor([ 0.0373,  0.0928,  0.0941,  ...,  0.0604,  0.0544, -0.0363])),\n",
       "             ('transformer.h.6.mlp.dense_h_to_4h.weight',\n",
       "              tensor([[-0.0039,  0.0248, -0.0087,  ...,  0.0057,  0.0389,  0.0001],\n",
       "                      [ 0.0099, -0.0486, -0.0018,  ...,  0.0122,  0.0257,  0.0392],\n",
       "                      [ 0.0030,  0.0040, -0.0050,  ..., -0.0024, -0.0102, -0.0114],\n",
       "                      ...,\n",
       "                      [ 0.0179, -0.0394,  0.0148,  ..., -0.0156,  0.0269,  0.0065],\n",
       "                      [ 0.0211, -0.0209,  0.0122,  ..., -0.0086, -0.0271,  0.0252],\n",
       "                      [-0.0051,  0.0145,  0.0117,  ..., -0.0161,  0.0257, -0.0146]])),\n",
       "             ('transformer.h.6.mlp.dense_h_to_4h.bias',\n",
       "              tensor([-0.3176, -0.0103, -0.0020,  ..., -0.1040, -0.4302, -0.0958])),\n",
       "             ('transformer.h.6.mlp.dense_4h_to_h.weight',\n",
       "              tensor([[ 0.0151, -0.0180,  0.0244,  ..., -0.0242, -0.0122,  0.0115],\n",
       "                      [-0.0042,  0.0107, -0.0022,  ..., -0.0052,  0.0066,  0.0080],\n",
       "                      [ 0.0010,  0.0026,  0.0094,  ..., -0.0136,  0.0037,  0.0126],\n",
       "                      ...,\n",
       "                      [ 0.0267, -0.0139,  0.0012,  ...,  0.0154, -0.0174,  0.0219],\n",
       "                      [-0.0227, -0.0144,  0.0228,  ..., -0.0017, -0.0194, -0.0032],\n",
       "                      [ 0.0041, -0.0040, -0.0087,  ...,  0.0057,  0.0312,  0.0028]])),\n",
       "             ('transformer.h.6.mlp.dense_4h_to_h.bias',\n",
       "              tensor([ 0.0210,  0.0253, -0.0928,  ..., -0.1941,  0.1486,  0.0008])),\n",
       "             ('transformer.h.7.input_layernorm.weight',\n",
       "              tensor([0.9229, 1.4424, 0.8486,  ..., 1.9102, 1.5566, 0.6411])),\n",
       "             ('transformer.h.7.input_layernorm.bias',\n",
       "              tensor([-0.0121,  0.0378,  0.1077,  ...,  0.0403, -0.0413,  0.0152])),\n",
       "             ('transformer.h.7.self_attention.query_key_value.weight',\n",
       "              tensor([[ 0.0772,  0.0050,  0.0316,  ..., -0.0025, -0.0400, -0.0172],\n",
       "                      [-0.0063, -0.0414,  0.0072,  ...,  0.0308, -0.0254, -0.0010],\n",
       "                      [ 0.0250, -0.0221,  0.0554,  ...,  0.0175,  0.0248, -0.0181],\n",
       "                      ...,\n",
       "                      [-0.0251, -0.0151,  0.0043,  ...,  0.0243,  0.0171,  0.0243],\n",
       "                      [-0.0412, -0.0037, -0.0022,  ..., -0.0317, -0.0029,  0.0150],\n",
       "                      [ 0.0024,  0.0273,  0.0017,  ..., -0.0161,  0.0141, -0.0106]])),\n",
       "             ('transformer.h.7.self_attention.query_key_value.bias',\n",
       "              tensor([ 2.4628e-02,  3.5645e-02, -7.1045e-02,  6.4026e-02, -6.3667e-03,\n",
       "                       5.1056e-02, -1.2833e-02,  5.8990e-02, -5.7983e-02, -1.0596e-01,\n",
       "                      -5.3040e-02, -2.6569e-03,  3.1281e-03,  2.2293e-02,  3.9032e-02,\n",
       "                      -1.4086e-03,  3.3600e-02, -4.9057e-03,  2.7664e-02, -1.6876e-02,\n",
       "                       2.1515e-02, -3.1082e-02, -3.4607e-02, -4.0741e-02,  2.6337e-02,\n",
       "                      -2.6505e-02, -6.5308e-02, -4.0558e-02,  3.1708e-02, -3.8483e-02,\n",
       "                       3.5980e-02,  5.2429e-02, -1.1429e-02,  3.1082e-02,  1.4061e-02,\n",
       "                       7.4654e-03,  1.0284e-01, -1.0239e-02, -6.0577e-02,  1.2726e-02,\n",
       "                      -6.0425e-02, -4.3976e-02, -2.0981e-02,  3.2928e-02, -4.8584e-02,\n",
       "                      -7.9803e-03,  2.9999e-02, -8.9025e-04, -8.9478e-02,  4.4708e-02,\n",
       "                       6.4819e-02, -4.8920e-02, -6.4209e-02,  4.3106e-03,  2.7145e-02,\n",
       "                       1.1467e-02,  7.3792e-02,  1.3870e-02, -1.1299e-02, -6.5063e-02,\n",
       "                      -6.5613e-02, -2.2186e-02,  2.4765e-02,  1.7075e-02,  2.2980e-02,\n",
       "                       4.7882e-02,  4.9133e-02,  2.7466e-03,  2.4734e-02,  4.2908e-02,\n",
       "                      -1.7487e-02,  3.7537e-02,  3.6774e-02, -4.8798e-02, -1.0724e-01,\n",
       "                       5.8167e-02, -5.6458e-02, -7.3059e-02, -4.2664e-02,  5.2521e-02,\n",
       "                       5.2917e-02, -6.2042e-02, -6.3965e-02, -7.7057e-03,  6.0944e-02,\n",
       "                       6.0791e-02,  5.3131e-02, -2.2171e-02, -5.1666e-02, -1.8448e-02,\n",
       "                      -1.0315e-01, -2.6352e-02,  4.9377e-02,  4.7974e-02,  1.7181e-02,\n",
       "                      -1.0386e-03,  5.1941e-02, -3.9482e-03, -5.7526e-02,  2.0477e-02,\n",
       "                       7.3303e-02,  8.5022e-02, -3.0045e-02, -4.6997e-03, -1.8539e-02,\n",
       "                       4.4922e-02,  4.5654e-02,  7.8918e-02,  4.5746e-02, -7.9102e-02,\n",
       "                      -6.3229e-04,  5.5298e-02, -4.3243e-02, -1.3145e-02,  9.6054e-03,\n",
       "                      -4.5868e-02,  4.8218e-03,  5.2917e-02, -1.4783e-01,  1.6266e-02,\n",
       "                       2.3331e-02,  1.5198e-01,  4.3091e-02,  2.1835e-02, -1.5845e-01,\n",
       "                      -1.4563e-01, -2.0542e-03, -2.2461e-02,  2.9358e-02,  3.8452e-02,\n",
       "                      -2.4414e-02,  3.6438e-02,  1.9821e-02,  1.2154e-02,  3.6438e-02,\n",
       "                       2.1561e-02, -7.4158e-02,  2.9251e-02,  1.6727e-03, -6.2790e-03,\n",
       "                      -4.5929e-03,  3.7994e-02,  1.6281e-02, -5.6976e-02,  3.9215e-02,\n",
       "                      -1.9943e-02, -5.0888e-03, -4.1290e-02, -1.4984e-02, -1.1482e-02,\n",
       "                      -9.6741e-03, -1.4324e-03,  2.3098e-03,  2.6688e-02,  2.5177e-02,\n",
       "                      -6.2065e-03, -1.5701e-02, -6.5765e-03,  1.8387e-02,  3.8086e-02,\n",
       "                       5.2216e-02, -4.7821e-02, -2.2141e-02,  3.0014e-02,  3.0350e-02,\n",
       "                      -6.8092e-03, -1.9867e-02, -5.6887e-04,  1.0536e-02,  2.7039e-02,\n",
       "                       1.0061e-03,  6.3599e-02, -1.6375e-03,  6.7810e-02, -5.8289e-03,\n",
       "                       1.0910e-02,  1.1803e-02,  9.8495e-03, -5.4596e-02, -1.1139e-03,\n",
       "                      -2.3743e-02,  2.5501e-03,  9.3765e-03, -3.5744e-03,  5.7587e-02,\n",
       "                      -5.6992e-03, -2.2980e-02,  2.2583e-03,  3.2654e-02,  3.7476e-02,\n",
       "                      -3.3081e-02, -4.9973e-03,  7.1838e-02,  1.1377e-01, -9.8694e-02,\n",
       "                       9.5581e-02,  1.1993e-01,  5.7404e-02, -3.7109e-02,  2.3193e-01,\n",
       "                      -1.1896e-01,  1.1725e-01, -1.5405e-01,  8.7646e-02, -4.8309e-02,\n",
       "                       4.9194e-02, -4.2145e-02,  1.1206e-03, -5.9143e-02, -6.3232e-02,\n",
       "                      -7.5745e-02,  8.5510e-02,  8.0872e-03,  1.5015e-02,  5.2917e-02,\n",
       "                      -3.4698e-02, -5.8929e-02, -1.6190e-02, -2.3035e-01, -1.0413e-01,\n",
       "                      -1.6084e+00,  2.1973e-02,  1.4380e-01,  1.0010e-01, -1.1725e-01,\n",
       "                       3.1921e-02, -8.5571e-02, -2.5684e-01, -2.2568e-02, -2.1094e-01,\n",
       "                       3.0579e-02,  1.0254e-02,  2.9221e-02, -3.6652e-02, -8.9050e-02,\n",
       "                       1.2585e-01,  1.4038e-01,  1.4636e-01,  3.2471e-02,  1.3628e-03,\n",
       "                      -4.4952e-02,  2.0056e-01,  1.2539e+00,  1.6571e-02,  1.8978e-03,\n",
       "                      -7.3059e-02, -1.7981e-01, -1.8567e-01,  9.1675e-02, -3.9185e-02,\n",
       "                      -1.6162e-01, -3.6084e-01,  1.5894e-01, -1.2311e-01, -1.3953e-01,\n",
       "                       1.3049e-01,  2.0004e-02, -4.9072e-02,  5.2338e-02,  4.1351e-02,\n",
       "                      -4.8523e-03,  4.8218e-02,  7.5806e-02, -1.2255e-03,  4.3427e-02,\n",
       "                       3.3722e-02, -5.1079e-03, -1.6678e-02,  6.5804e-03, -6.6261e-03,\n",
       "                      -1.4954e-03,  2.0752e-02,  2.5635e-02, -1.2655e-03, -1.9653e-02,\n",
       "                       6.6040e-02,  2.9419e-02, -1.4511e-02, -1.4687e-02, -1.8143e-02,\n",
       "                      -3.8055e-02, -4.1168e-02,  5.0110e-02,  8.8989e-02, -4.1504e-02,\n",
       "                       1.3351e-03,  2.6398e-02,  7.0305e-03, -1.2573e-02, -4.3411e-03,\n",
       "                      -2.7466e-02,  6.4621e-03,  3.2902e-03, -2.7218e-03, -2.5034e-04,\n",
       "                      -1.3557e-02, -6.4697e-03, -1.4503e-02, -1.7517e-02, -1.5503e-02,\n",
       "                      -1.9714e-02, -3.2806e-02, -2.5452e-02, -7.0343e-03,  8.5220e-03,\n",
       "                       1.7868e-02,  1.3054e-02, -4.6906e-02,  1.7014e-02, -2.0584e-02,\n",
       "                       6.9824e-02,  6.9656e-03, -2.7405e-02, -4.6501e-03,  1.4069e-02,\n",
       "                      -4.8943e-03, -3.3740e-01,  2.2141e-02, -1.2505e-02,  8.3847e-03,\n",
       "                      -1.8680e-04,  7.0419e-03, -3.7567e-02,  8.7280e-03, -9.4986e-03,\n",
       "                       2.6264e-03, -5.4321e-03,  7.8249e-04, -1.8555e-02, -1.4900e-02,\n",
       "                      -4.1542e-03,  3.6488e-03, -1.3382e-02,  1.4549e-02,  2.6073e-03,\n",
       "                      -1.0675e-01, -2.3071e-02,  1.1454e-03, -8.3069e-02, -1.3176e-02,\n",
       "                      -4.0955e-02,  5.0774e-03, -2.4471e-03,  9.6588e-03,  2.1713e-02,\n",
       "                      -3.1799e-02,  7.8308e-02, -1.0834e-02, -3.5553e-02, -1.7624e-02,\n",
       "                      -1.7181e-02,  2.1286e-02, -6.8970e-03,  1.3931e-02, -3.7689e-02,\n",
       "                       4.5586e-03, -1.6983e-02, -1.1909e-02, -9.3307e-03,  8.0795e-03,\n",
       "                       7.5912e-03, -1.2680e-02, -1.1810e-02, -6.3248e-03, -3.5267e-03,\n",
       "                      -5.1804e-03,  1.5190e-02,  4.1687e-02,  4.3945e-02,  5.3596e-03,\n",
       "                       2.0020e-02, -1.3802e-02, -2.2415e-02,  4.2076e-03, -1.2634e-02,\n",
       "                      -8.0338e-03,  2.0081e-02,  2.3636e-02, -9.3307e-03, -2.4353e-02,\n",
       "                      -1.7517e-02,  1.5942e-01,  7.4539e-03,  1.5900e-02, -1.3818e-01,\n",
       "                       1.8335e-01, -4.1943e-01, -6.6162e-02,  2.6709e-01, -1.7566e-01,\n",
       "                      -3.4546e-02,  3.5181e-01,  1.1780e-02, -2.3682e-01, -1.6211e-01,\n",
       "                       6.6936e-05, -1.4636e-01,  8.6243e-02, -2.8735e-01,  2.7246e-01,\n",
       "                       4.2310e-01,  1.2676e+00, -1.6394e-01, -2.4643e-02, -4.1260e-01,\n",
       "                       9.6313e-02,  8.4961e-02,  1.5259e-01, -3.8721e-01, -3.1860e-02,\n",
       "                      -4.4385e-01,  2.9248e-01, -8.3679e-02,  1.1188e-01,  1.8652e-01,\n",
       "                      -9.4849e-02,  6.3721e-02,  4.4604e-01, -1.1787e+00,  1.1029e-01,\n",
       "                       3.2495e-01, -1.2891e-01,  1.8237e-01,  1.0117e+00, -3.1586e-02,\n",
       "                       1.7810e-01, -3.0640e-02, -4.0833e-02, -1.0620e-01, -1.6211e-01,\n",
       "                       1.1633e-01,  2.2156e-01, -2.0911e-01, -1.9043e-01, -1.1194e-01,\n",
       "                      -1.8906e-02,  3.1021e-02,  4.6021e-02,  2.3047e-01, -1.4185e-01,\n",
       "                       2.0520e-01, -3.7866e-01,  3.9177e-03, -3.3154e-01,  2.1777e-01,\n",
       "                      -3.7695e-01,  2.8896e-03,  1.3271e+00, -4.9414e-01, -8.6963e-01,\n",
       "                       1.0681e-01,  1.8335e-01,  3.1982e-01, -1.2604e-02,  1.4502e+00,\n",
       "                       1.0771e+00, -2.6318e-01,  1.8384e-01, -3.9819e-01, -2.0801e-01,\n",
       "                      -1.2875e-03,  1.1328e+00, -7.1387e-01,  5.3271e-01,  2.0374e-01,\n",
       "                       2.8662e-01, -1.0358e-01, -4.3750e-01, -3.3472e-01,  8.2550e-03,\n",
       "                       2.8125e-01,  2.5171e-01, -2.6733e-01, -1.4414e+00, -3.7793e-01,\n",
       "                      -1.7670e-02,  1.7639e-01, -1.0869e+00,  2.0081e-01,  1.7609e-02,\n",
       "                      -2.2729e-01,  3.8452e-02,  4.7485e-02, -1.0828e-01, -6.3623e-01,\n",
       "                      -7.2803e-01, -1.9507e-01,  1.0381e+00, -5.5237e-03, -1.4014e-01,\n",
       "                       2.4084e-01,  2.8467e-01,  3.7158e-01, -2.3962e-01, -5.0342e-01,\n",
       "                       1.6846e-01,  3.1543e-01, -1.2188e+00,  1.1797e+00, -7.3584e-01,\n",
       "                       9.7229e-02,  7.5488e-01,  1.2366e-01, -1.6272e-01,  6.6113e-01,\n",
       "                       8.8928e-02,  7.1045e-01, -5.6055e-01,  4.4824e-01, -5.3711e-02,\n",
       "                       7.9407e-02,  2.6074e-01,  9.4727e-02, -7.4890e-02,  2.1637e-02,\n",
       "                      -2.7612e-01, -1.0430e+00, -4.8706e-01, -3.0762e-01,  5.0391e-01,\n",
       "                      -5.1367e-01,  6.1310e-02, -1.3501e-01,  1.2683e-01,  6.4697e-02,\n",
       "                      -4.4312e-02, -2.1399e-01, -1.4107e-02, -6.7871e-02, -1.0858e-01,\n",
       "                      -1.6223e-01,  2.3486e-01, -2.7271e-01,  3.6774e-02, -9.1492e-02,\n",
       "                       2.1667e-01, -1.8982e-02,  9.2529e-02, -1.2183e-01, -3.7720e-02,\n",
       "                      -5.0000e-01,  1.2878e-01, -5.3436e-02, -1.1493e-01, -9.0576e-02,\n",
       "                       1.0132e-01,  2.1997e-01, -8.6548e-02, -1.6821e-01,  4.4159e-02,\n",
       "                       9.6924e-02, -5.3375e-02, -9.8022e-02,  2.9150e-01, -9.8572e-02,\n",
       "                      -3.2928e-02,  3.3472e-01, -1.4258e-01, -7.9956e-02, -1.0117e+00,\n",
       "                       2.7496e-02, -1.0547e-01,  5.9357e-02, -3.1055e-01,  5.1465e-01,\n",
       "                      -9.7656e-02, -2.7122e-03, -2.9404e-02,  1.5068e-02,  3.3740e-01,\n",
       "                      -1.1143e+00,  9.8438e-01,  1.2708e-01,  1.8896e-01,  6.1615e-02,\n",
       "                      -1.0381e+00, -1.6533e+00,  6.6260e-01, -9.8389e-01,  3.1201e-01,\n",
       "                      -1.7939e+00,  3.6548e-01, -1.1273e-01, -1.4685e-01,  3.5248e-02,\n",
       "                      -2.8809e-01,  2.0239e-01, -4.7974e-02, -6.2408e-02, -1.7646e+00,\n",
       "                      -4.4482e-01,  1.8857e+00,  6.7822e-01,  2.8760e-01,  8.5938e-02,\n",
       "                      -1.8936e+00, -3.9673e-01, -6.4392e-03,  8.2031e-01, -6.4307e-01,\n",
       "                       4.0356e-01,  1.3838e+00, -6.9971e-01,  1.8105e+00,  1.8945e-01,\n",
       "                      -1.3464e-01,  1.7412e+00,  2.2913e-01,  3.5059e-01, -4.6069e-01,\n",
       "                      -4.6478e-02,  2.6074e-01, -1.7783e+00,  1.8237e-01, -3.5376e-01,\n",
       "                      -1.0754e-01,  5.9814e-01,  1.9519e-01, -4.3286e-01, -1.8542e-01,\n",
       "                       1.7686e+00,  2.4194e-01, -2.0483e-01, -5.8105e-01,  2.3181e-01,\n",
       "                      -1.6631e+00, -7.4072e-01,  5.1611e-01, -4.6997e-01,  1.6689e+00,\n",
       "                       1.1658e-01,  1.1192e-02, -7.5293e-01, -1.0645e+00, -4.6704e-01,\n",
       "                      -1.8762e-01,  2.9590e-01,  3.5864e-01, -1.0312e+00, -6.5186e-01,\n",
       "                      -2.5031e+01,  2.9199e+00, -1.1477e+01,  9.3906e+00, -2.9531e+01,\n",
       "                       5.4023e+00,  5.2852e+00,  1.4492e+00, -4.2773e+00, -1.8291e+00,\n",
       "                       2.5918e+00,  5.5469e-01,  3.5781e+00, -2.7109e+01, -1.9805e+00,\n",
       "                       3.0547e+01,  5.7812e+00, -3.5801e+00,  2.4590e+00, -3.0047e+01,\n",
       "                       2.3834e-02, -4.7974e-01,  9.2109e+00, -8.8516e+00,  2.2969e+00,\n",
       "                       1.9172e+01, -1.0641e+01,  3.0781e+01, -1.0107e+00, -4.8608e-01,\n",
       "                       2.7125e+01,  2.9785e-01, -4.0210e-01, -9.7852e-01,  5.1855e-01,\n",
       "                       6.8672e+00, -2.7641e+01, -3.3047e+00, -6.6094e+00, -4.8359e+00,\n",
       "                      -2.5000e-01,  1.7637e+00, -7.5625e+00, -5.5938e+00,  2.5828e+01,\n",
       "                       2.5293e+00, -6.6836e+00, -8.8203e+00,  1.4854e+00, -2.8125e+01,\n",
       "                      -9.7344e+00,  7.7930e+00,  3.6279e-01,  2.6641e+01,  2.8359e+00,\n",
       "                      -3.3848e+00, -1.2102e+01, -1.6391e+01, -4.3867e+00,  2.9648e+00,\n",
       "                       6.1797e+00, -2.8184e+00, -1.2648e+01, -3.7812e+00,  3.0060e-02,\n",
       "                      -2.1500e-02,  2.4307e-02, -1.6525e-02, -3.6041e-02,  1.8250e-02,\n",
       "                       4.0894e-02, -3.1967e-03, -2.5610e-01,  2.7197e-01,  7.9489e-04,\n",
       "                       4.4281e-02, -7.7934e-03, -2.9419e-02, -1.3336e-02,  1.3647e-01,\n",
       "                      -1.9974e-02, -7.9895e-02,  4.1504e-02,  6.3171e-02,  2.6840e-02,\n",
       "                       3.6316e-02, -7.0740e-02, -2.2629e-02, -1.1066e-01, -3.7766e-04,\n",
       "                       8.3496e-02, -2.9297e-02, -3.9825e-02,  1.0513e-02,  4.4327e-03,\n",
       "                      -9.6130e-03, -5.3680e-02, -6.8909e-02,  8.5640e-04, -8.9844e-02,\n",
       "                      -2.2278e-02,  2.3499e-02,  1.7136e-02, -2.1118e-02, -2.0349e-01,\n",
       "                      -7.1411e-02,  1.4014e-01,  3.0880e-03, -1.6098e-02,  8.0948e-03,\n",
       "                      -3.2745e-02, -1.0338e-02, -9.1076e-04, -5.6274e-02,  6.2485e-03,\n",
       "                      -1.2993e-02,  3.6755e-03,  1.1187e-03,  2.8961e-02, -1.2894e-02,\n",
       "                      -2.5726e-02, -3.6469e-03, -7.5531e-03,  2.8412e-02,  2.0767e-02,\n",
       "                      -7.8087e-03, -1.4320e-02, -1.8738e-02])),\n",
       "             ('transformer.h.7.self_attention.dense.weight',\n",
       "              tensor([[ 0.0063, -0.0033, -0.0059,  ..., -0.0030, -0.0183, -0.0166],\n",
       "                      [-0.0103,  0.0116, -0.0145,  ...,  0.0101,  0.0069, -0.0136],\n",
       "                      [ 0.0025,  0.0118, -0.0116,  ..., -0.0114, -0.0277, -0.0222],\n",
       "                      ...,\n",
       "                      [-0.0318, -0.0029,  0.0039,  ..., -0.0111,  0.0057, -0.0018],\n",
       "                      [ 0.0135, -0.0135,  0.0038,  ..., -0.0202, -0.0127,  0.0130],\n",
       "                      [-0.0110,  0.0068,  0.0071,  ..., -0.0012,  0.0121, -0.0112]])),\n",
       "             ('transformer.h.7.self_attention.dense.bias',\n",
       "              tensor([-0.1293,  0.0706,  0.0988,  ..., -0.2969, -0.1320, -0.0089])),\n",
       "             ('transformer.h.7.post_attention_layernorm.weight',\n",
       "              tensor([0.9507, 1.2148, 0.7681,  ..., 1.6094, 1.3252, 0.5146])),\n",
       "             ('transformer.h.7.post_attention_layernorm.bias',\n",
       "              tensor([ 0.0572,  0.0614,  0.1655,  ..., -0.0084, -0.0351, -0.0172])),\n",
       "             ('transformer.h.7.mlp.dense_h_to_4h.weight',\n",
       "              tensor([[ 0.0085, -0.0082,  0.0021,  ...,  0.0071, -0.0117, -0.0013],\n",
       "                      [-0.0423,  0.0069,  0.0383,  ..., -0.0255,  0.0421,  0.0095],\n",
       "                      [ 0.0271, -0.0078,  0.0191,  ...,  0.0133, -0.0071, -0.0028],\n",
       "                      ...,\n",
       "                      [-0.0053,  0.0049, -0.0091,  ...,  0.0460,  0.0102, -0.0262],\n",
       "                      [-0.0087,  0.0001,  0.0123,  ...,  0.0187,  0.0209, -0.0147],\n",
       "                      [-0.0354,  0.0437, -0.0201,  ..., -0.0071,  0.0128,  0.0272]])),\n",
       "             ('transformer.h.7.mlp.dense_h_to_4h.bias',\n",
       "              tensor([ 0.0581, -0.4780,  0.0022,  ..., -0.5518, -0.5186, -0.3259])),\n",
       "             ('transformer.h.7.mlp.dense_4h_to_h.weight',\n",
       "              tensor([[-0.0058, -0.0367, -0.0120,  ...,  0.0001, -0.0141, -0.0127],\n",
       "                      [ 0.0106,  0.0024,  0.0357,  ..., -0.0002,  0.0249,  0.0007],\n",
       "                      [-0.0008,  0.0080,  0.0089,  ..., -0.0014, -0.0091, -0.0024],\n",
       "                      ...,\n",
       "                      [-0.0206, -0.0287, -0.0462,  ...,  0.0123,  0.0117, -0.0157],\n",
       "                      [ 0.0071,  0.0369, -0.0089,  ...,  0.0074,  0.0324, -0.0191],\n",
       "                      [-0.0093, -0.0091, -0.0003,  ...,  0.0019,  0.0031, -0.0007]])),\n",
       "             ('transformer.h.7.mlp.dense_4h_to_h.bias',\n",
       "              tensor([ 0.1654, -0.0437, -0.1959,  ..., -0.0853,  0.1526,  0.0202])),\n",
       "             ('transformer.h.8.input_layernorm.weight',\n",
       "              tensor([1.2256, 1.7637, 1.0283,  ..., 2.3496, 1.8730, 0.7505])),\n",
       "             ('transformer.h.8.input_layernorm.bias',\n",
       "              tensor([ 0.0493,  0.0362,  0.1060,  ...,  0.0404, -0.0242,  0.0238])),\n",
       "             ('transformer.h.8.self_attention.query_key_value.weight',\n",
       "              tensor([[-0.0156,  0.0293, -0.0194,  ..., -0.0073, -0.0299, -0.0184],\n",
       "                      [-0.0371, -0.0270,  0.0122,  ...,  0.0236, -0.0057, -0.0387],\n",
       "                      [-0.0127,  0.0069,  0.0598,  ..., -0.0007, -0.0044, -0.0101],\n",
       "                      ...,\n",
       "                      [-0.0038, -0.0013, -0.0146,  ...,  0.0081, -0.0073, -0.0085],\n",
       "                      [ 0.0477,  0.0081,  0.0058,  ...,  0.0201,  0.0239, -0.0075],\n",
       "                      [-0.0267, -0.0105,  0.0113,  ..., -0.0247, -0.0223, -0.0124]])),\n",
       "             ('transformer.h.8.self_attention.query_key_value.bias',\n",
       "              tensor([ 5.5145e-02, -1.0114e-01, -1.8829e-02,  3.8330e-02,  2.8107e-02,\n",
       "                      -1.8951e-02,  2.0828e-03,  6.1462e-02,  4.6600e-02, -1.7725e-01,\n",
       "                       1.2732e-01, -1.7605e-03,  4.5654e-02,  1.8600e-02,  4.7028e-02,\n",
       "                       1.0223e-01,  7.6111e-02, -1.3049e-01, -1.1787e-02,  3.7842e-02,\n",
       "                      -8.3374e-02,  1.2617e+00, -6.5430e-02,  2.6138e-02, -3.1799e-02,\n",
       "                      -5.9143e-02, -1.7041e-01, -1.2842e-01, -7.2266e-02, -3.8223e-03,\n",
       "                       2.4231e-02, -1.1108e-01, -1.3781e-03, -1.1497e-02,  4.6143e-02,\n",
       "                      -6.0997e-03,  5.2887e-02, -6.3610e-04,  5.4932e-02,  4.9324e-03,\n",
       "                       2.6199e-02, -5.1483e-02,  3.5065e-02,  1.9159e-03, -1.9531e-02,\n",
       "                       3.3600e-02, -1.1639e-01, -8.3984e-02,  2.4338e-02,  2.0096e-02,\n",
       "                      -9.3567e-02,  3.8940e-02, -1.2722e-03, -1.2489e-02, -7.4524e-02,\n",
       "                       1.2184e-02,  1.5854e-02,  9.0149e-02,  6.8909e-02,  5.7220e-02,\n",
       "                       2.7985e-02,  1.7044e-02, -1.0681e-02,  5.9906e-02, -2.6199e-02,\n",
       "                       8.9661e-02,  2.0947e-01,  1.3039e-02,  4.0039e-02, -4.2511e-02,\n",
       "                       1.3220e-01, -1.3831e-01,  4.1229e-02, -1.3110e-01,  9.7595e-02,\n",
       "                      -6.0822e-02,  3.5645e-02,  7.2693e-02, -1.0626e-01,  1.6203e-03,\n",
       "                      -9.1309e-01,  3.5309e-02,  9.9670e-02, -6.6681e-03,  1.4771e-02,\n",
       "                       1.1658e-01,  2.1469e-02, -4.3915e-02, -3.2928e-02,  9.3445e-02,\n",
       "                      -2.7344e-02, -1.0004e-01,  1.0272e-01, -2.2766e-02, -1.3748e-02,\n",
       "                      -5.3902e-03,  1.1551e-02,  2.2079e-02, -1.0468e-01,  3.3594e-01,\n",
       "                      -9.0393e-02,  9.0271e-02,  1.2769e-01, -8.1177e-02,  1.5297e-02,\n",
       "                       1.1371e-01,  1.4758e-01,  8.0750e-02,  7.3669e-02, -1.2146e-02,\n",
       "                      -7.5134e-02,  1.5906e-01,  4.0039e-02,  1.6327e-02, -9.3384e-03,\n",
       "                      -1.4331e-01, -4.6883e-03,  3.5034e-02,  3.6469e-02, -1.0979e-02,\n",
       "                      -1.2561e-01, -4.6783e-02, -6.1462e-02, -1.1230e-01, -9.0332e-02,\n",
       "                       5.2185e-02,  4.0222e-02, -7.3669e-02,  3.8300e-02,  1.5732e-02,\n",
       "                       1.9257e-02,  4.1437e-04, -1.1909e-02,  3.3474e-03,  2.2202e-03,\n",
       "                      -2.3560e-02,  1.8646e-02,  1.1726e-02, -8.9874e-03, -3.1052e-02,\n",
       "                       1.6823e-03, -2.1179e-02, -1.3847e-02,  1.5625e-02,  2.7435e-02,\n",
       "                      -1.9287e-02, -2.1194e-02, -2.8061e-02, -4.1870e-02,  2.6520e-02,\n",
       "                      -2.8324e-03,  1.7059e-02, -3.1929e-03, -1.8692e-02,  1.2077e-02,\n",
       "                      -8.6288e-03, -9.6817e-03,  1.0994e-02, -4.5395e-03, -7.7171e-03,\n",
       "                      -2.7103e-03,  2.2644e-02, -1.2465e-03,  4.0855e-03,  2.3300e-02,\n",
       "                       9.9564e-03,  1.2207e-02,  1.8005e-02, -5.7487e-03, -4.6570e-02,\n",
       "                      -2.5482e-03, -3.3936e-02,  1.4267e-02,  4.5898e-02,  1.5593e-03,\n",
       "                      -3.8605e-03, -2.5964e-04, -3.1738e-02,  1.4755e-02, -2.1576e-02,\n",
       "                      -4.9515e-03, -1.9730e-02, -1.4626e-02,  2.6493e-03,  2.7523e-03,\n",
       "                       2.4536e-02, -6.6757e-03,  3.6591e-02, -1.9104e-02,  1.5144e-02,\n",
       "                      -1.8707e-02,  1.6832e-03,  1.0689e-02,  7.4951e-02, -1.0895e-01,\n",
       "                       1.8140e-01, -4.5990e-02,  2.4429e-02, -3.0777e-02,  4.0314e-02,\n",
       "                      -2.0227e-01, -1.3599e-01, -8.8379e-02,  4.5502e-02,  1.2964e-01,\n",
       "                       1.8652e-01,  1.5027e-01, -9.5581e-02, -5.1636e-02,  4.7150e-02,\n",
       "                      -2.8955e-01,  6.0425e-02, -7.9880e-03, -1.5526e-02, -1.0712e-01,\n",
       "                      -3.3911e-01,  3.1708e-02,  4.7836e-03,  3.3508e-02,  2.0416e-02,\n",
       "                      -7.2632e-02,  8.5571e-02,  7.0007e-02, -7.5769e-04,  1.7822e-02,\n",
       "                       9.1919e-02,  6.5576e-01, -1.7371e-01,  1.7041e-01,  2.5562e-01,\n",
       "                      -1.4189e+00, -1.6708e-02, -1.2123e-02,  2.8589e-01, -8.5693e-02,\n",
       "                      -2.1960e-01,  3.3112e-03,  1.6760e-01, -1.8555e-01, -7.4341e-02,\n",
       "                       4.2786e-02, -8.6609e-02, -7.1594e-02,  1.6785e-01,  7.7209e-02,\n",
       "                       1.9006e-01,  5.0354e-02, -1.6602e-02, -1.5771e-01, -1.4661e-01,\n",
       "                      -2.1289e-01, -7.3853e-02, -5.0049e-02,  2.1805e-02,  2.0483e-01,\n",
       "                       1.6895e-01, -2.2030e-03, -2.3178e-02, -1.8021e-02, -1.2016e-02,\n",
       "                       2.9877e-02, -7.0190e-03,  1.3838e-03,  9.9335e-03,  8.1778e-04,\n",
       "                      -1.7731e-02, -1.3542e-02, -2.6001e-01, -1.1055e-02,  3.6407e-02,\n",
       "                       5.4398e-03, -2.6505e-02,  2.1687e-03, -2.1271e-02,  4.5815e-03,\n",
       "                       1.1826e-02, -3.1281e-02,  6.3553e-03,  1.3145e-02, -3.0960e-02,\n",
       "                      -4.7493e-03, -9.0179e-03, -3.3173e-02,  1.3763e-02,  5.3024e-03,\n",
       "                      -4.7660e-04, -2.4063e-02,  2.0157e-02,  1.6251e-02, -1.6356e-03,\n",
       "                       2.8137e-02, -2.8000e-02,  8.1396e-04,  8.5831e-03,  2.1378e-02,\n",
       "                      -8.5220e-03, -6.5308e-03,  2.3346e-02, -2.1759e-02,  3.2745e-02,\n",
       "                       2.1420e-03,  2.8900e-02,  2.9633e-02,  3.1033e-03, -2.0264e-02,\n",
       "                       2.6337e-02,  3.0182e-02,  4.9400e-03, -4.0131e-03,  1.1620e-02,\n",
       "                       1.4343e-02, -1.5268e-03,  7.8888e-03, -5.6534e-03,  1.3206e-02,\n",
       "                      -5.8212e-03,  1.7639e-02, -9.5291e-03, -1.2024e-02,  4.6310e-03,\n",
       "                      -5.6000e-03, -9.4757e-03, -9.8877e-03,  2.7771e-03, -9.8114e-03,\n",
       "                      -6.4926e-03,  1.6384e-03, -3.6030e-03,  1.4336e-02, -1.0368e-02,\n",
       "                       2.1591e-02, -7.2289e-03,  2.0828e-02, -5.4893e-03,  6.5346e-03,\n",
       "                      -2.5757e-02, -2.2156e-02,  8.4457e-03,  7.4654e-03,  9.5673e-03,\n",
       "                       1.4906e-03, -1.5472e-02,  2.1057e-02,  1.0712e-02, -1.0780e-02,\n",
       "                       2.6131e-03, -6.8321e-03, -1.6613e-03, -6.3705e-03,  5.4398e-03,\n",
       "                      -5.0507e-03,  9.4910e-03,  1.8902e-03,  6.0196e-03, -1.3870e-02,\n",
       "                       1.5497e-03, -1.5465e-02, -4.6196e-03, -3.6831e-03, -1.0862e-03,\n",
       "                      -1.5587e-02,  1.1070e-02,  3.4070e-04, -7.2205e-02,  2.3117e-03,\n",
       "                      -2.0950e-02, -1.5287e-03, -1.1993e-02,  3.3283e-03, -6.0501e-03,\n",
       "                      -4.6806e-03, -2.5787e-03, -7.2622e-04,  2.0401e-02, -2.4426e-01,\n",
       "                      -1.2276e-02, -1.0900e-03, -1.6623e-03, -5.6458e-04,  1.6205e-02,\n",
       "                      -5.6419e-03,  4.2038e-03,  4.5586e-03, -2.3098e-03,  2.8662e-01,\n",
       "                       5.7678e-02, -9.4434e-01,  1.8445e-01, -3.7567e-02,  3.3936e-02,\n",
       "                       9.7900e-01, -9.2822e-01, -1.7273e-02, -1.5244e-02,  1.0474e-01,\n",
       "                       3.6640e-03,  1.1639e-01, -8.2129e-01, -7.2327e-02,  4.3488e-02,\n",
       "                       8.4656e-02, -5.2223e-03, -4.1901e-02, -7.8247e-02,  1.0663e-01,\n",
       "                       1.4990e-01, -9.4922e-01, -5.3760e-01,  6.0059e-02, -2.6871e-02,\n",
       "                      -6.7993e-02,  9.8877e-02,  9.3079e-02, -1.3733e-01,  1.9568e-01,\n",
       "                       1.5356e-01,  5.2704e-02,  7.2449e-02, -9.4299e-02,  8.7036e-02,\n",
       "                       7.6477e-02, -2.2266e-01, -8.4106e-02,  7.1594e-02,  2.9102e-01,\n",
       "                       2.0325e-02, -6.2103e-02,  1.0834e-01, -8.4167e-02,  1.7554e-01,\n",
       "                       4.5319e-02,  6.8237e-02, -9.5166e-01,  6.5422e-03, -1.5637e-01,\n",
       "                       1.0195e+00, -2.9434e-02, -1.0449e+00,  5.0873e-02, -1.2421e-02,\n",
       "                      -2.6660e-01, -9.0179e-03,  7.6965e-02,  6.3867e-01,  2.4353e-02,\n",
       "                      -5.0000e-01, -9.9258e-03,  8.6487e-02, -2.6123e-01, -1.4209e-01,\n",
       "                       8.5156e-01, -4.9805e-02,  3.1274e-01, -3.3838e-01, -9.8877e-01,\n",
       "                       8.2764e-01, -1.1914e-01,  3.0469e-01, -1.5466e-01,  5.0879e-01,\n",
       "                       1.1438e-01,  7.9053e-01,  4.5020e-01,  1.5601e-01,  1.6077e-01,\n",
       "                      -1.6650e-01, -1.6919e-01,  2.2961e-01, -1.6992e-01, -2.1057e-01,\n",
       "                       7.5732e-01,  5.9082e-01, -2.5830e-01, -4.1357e-01, -1.7395e-01,\n",
       "                      -5.5078e-01, -1.8372e-01,  1.7593e-02, -4.0601e-01,  4.1553e-01,\n",
       "                      -2.9468e-01, -2.3242e-01,  1.9501e-02,  2.0142e-01,  5.6299e-01,\n",
       "                       3.2959e-01,  1.0669e-01, -5.8899e-02, -3.4741e-01, -1.2659e-01,\n",
       "                       3.8574e-01, -2.0496e-01, -3.4375e-01, -5.9277e-01, -2.6291e-02,\n",
       "                      -1.7792e-02,  8.7500e-01, -2.0645e-02, -1.5613e-01, -8.4912e-01,\n",
       "                       1.4478e-01,  9.8242e-01, -1.5295e-01,  6.0254e-01,  3.8135e-01,\n",
       "                      -1.9080e-01,  3.4058e-01, -1.3613e+00,  2.2812e-02,  6.9336e-01,\n",
       "                      -4.3091e-01, -4.8730e-01,  1.0000e+00,  1.9006e-01, -1.1270e+00,\n",
       "                       9.5215e-01, -2.1606e-02, -5.7556e-02, -9.5032e-02, -4.1809e-02,\n",
       "                       1.4512e+00,  9.4223e-03,  3.7079e-02, -1.1045e+00, -1.1680e+00,\n",
       "                       8.4229e-02,  1.9800e-01,  8.7341e-02,  6.6992e-01,  1.1694e-01,\n",
       "                      -5.2832e-01, -2.2491e-02,  1.2500e+00,  1.1621e+00, -8.4229e-02,\n",
       "                       5.3162e-02, -1.3452e-01, -9.6387e-01,  3.1250e-02,  7.8552e-02,\n",
       "                       1.2695e+00,  4.6326e-02, -2.5757e-01, -4.1107e-02, -7.6807e-01,\n",
       "                       6.2164e-02, -2.2827e-01, -1.0266e-01,  3.6401e-01,  9.1675e-02,\n",
       "                      -1.2471e+00,  1.0541e-01,  1.2627e+00, -3.2056e-01, -1.5942e-01,\n",
       "                       3.7231e-01, -1.2158e-01, -1.7960e-02, -7.9679e-04,  9.4971e-01,\n",
       "                      -3.4363e-02, -4.2554e-01, -1.3838e+00, -4.3384e-01,  1.4441e-01,\n",
       "                      -2.0462e-02,  2.4915e-01,  1.1215e-02,  2.0532e-01, -1.2347e-01,\n",
       "                       9.8022e-02, -1.3193e+00,  4.7150e-03, -4.3701e-02, -6.0742e-01,\n",
       "                       1.3398e+00,  5.4688e-02, -7.4883e-03, -6.7993e-02, -7.2388e-02,\n",
       "                       8.0322e-01,  2.6207e-03, -4.1779e-02,  1.6222e-03, -2.7908e-02,\n",
       "                       2.7267e-02, -1.2036e-01, -2.2263e-02, -1.4381e-02, -8.4424e-01,\n",
       "                      -1.1609e-01, -1.9669e-02, -2.6733e-02,  9.2090e-01, -6.2927e-02,\n",
       "                       1.1896e-01,  7.6111e-02, -3.6987e-02, -4.1016e-02,  1.2030e-01,\n",
       "                       1.0016e-01, -2.9321e-01, -5.1074e-01, -3.8490e-03,  4.8370e-02,\n",
       "                       1.2256e-01, -7.4854e-01, -2.7328e-02, -1.4259e-02, -6.9885e-03,\n",
       "                       1.2505e-02, -1.4355e-01, -1.2390e-02, -5.9776e-03,  3.7781e-02,\n",
       "                      -2.8946e-02,  1.2549e-01, -1.9104e-02, -2.9640e-03, -1.3049e-01,\n",
       "                       1.0388e-01, -1.4076e-02, -3.5461e-02,  7.3425e-02, -7.2998e-02,\n",
       "                       1.8951e-02,  6.1963e-01, -4.4037e-02, -5.1910e-02,  1.4905e-01,\n",
       "                       3.5614e-02,  3.8635e-02,  9.8328e-02,  9.9060e-02, -4.9805e-01,\n",
       "                      -9.4700e-04, -4.1199e-02,  5.8807e-02,  2.0844e-02,  7.2861e-03,\n",
       "                      -1.1511e-01,  6.2061e-01, -2.4811e-02,  2.5684e+00, -8.1172e+00,\n",
       "                       1.8408e-01,  8.2324e-01,  1.9102e+00,  1.4775e+00,  8.4033e-01,\n",
       "                       2.3608e-01, -1.2854e-01,  4.7339e-01,  8.5625e+00,  4.5361e-01,\n",
       "                       3.8257e-01,  4.6753e-01, -9.5547e+00,  1.5508e+00, -2.3906e+00,\n",
       "                      -1.3652e+00,  4.7607e-01,  8.1641e-01,  4.0869e-01, -1.0020e+00,\n",
       "                       3.6973e+00,  6.3750e+00,  4.5752e-01,  2.5781e-01, -9.6875e-01,\n",
       "                       7.9805e+00,  1.0322e+00, -5.9814e-01,  1.5417e-01,  3.8574e-01,\n",
       "                       2.0293e+00, -5.9766e-01, -8.3545e-01, -1.7615e-01,  9.6729e-01,\n",
       "                      -3.3105e-01,  8.5510e-02,  5.3436e-02,  1.6182e+00,  1.4465e-02,\n",
       "                      -9.3604e-01,  3.7085e-01, -1.4854e+00,  1.0215e+00, -3.4363e-02,\n",
       "                      -7.3672e+00, -9.6802e-02,  1.3164e+00, -4.2227e+00, -2.4551e+00,\n",
       "                      -1.3203e+00, -2.5957e+00, -1.6074e+00,  6.2305e+00, -4.4946e-01,\n",
       "                      -7.2412e-01, -1.7734e+00, -2.0723e+00,  5.8936e-01, -3.6774e-02,\n",
       "                       3.0708e-03, -1.2001e-02,  1.9388e-03, -2.3239e-02, -1.2238e-02,\n",
       "                       1.5579e-02,  7.1106e-03,  2.8763e-03, -2.6855e-03,  1.5945e-02,\n",
       "                      -5.7335e-03, -1.2245e-03, -4.0665e-03,  1.4183e-02,  3.7594e-03,\n",
       "                      -5.3453e-04, -2.4429e-02,  2.1347e-02, -5.6458e-04,  1.6487e-04,\n",
       "                       1.3247e-03, -1.9588e-03, -2.0050e-02,  2.4738e-03,  1.0178e-02,\n",
       "                      -9.7504e-03, -1.0384e-02, -7.0992e-03, -5.5504e-03, -5.6267e-03,\n",
       "                       2.8000e-03, -2.6688e-02, -1.1864e-02, -2.6245e-02,  2.2278e-02,\n",
       "                      -7.9727e-03, -6.2943e-03, -1.1307e-02,  1.1072e-01,  4.6082e-03,\n",
       "                       5.9662e-03, -2.5101e-03,  1.4435e-02, -2.0447e-02,  1.5955e-03,\n",
       "                      -1.2428e-02,  9.2163e-03, -2.3041e-03,  3.5034e-02,  4.4465e-04,\n",
       "                       3.8280e-03,  6.7673e-03,  3.4668e-02,  1.6443e-01,  1.4145e-02,\n",
       "                      -2.5578e-03,  1.1772e-02, -1.1444e-02, -1.2321e-02, -4.3182e-03,\n",
       "                       1.1124e-02,  1.8570e-02, -7.0740e-02])),\n",
       "             ('transformer.h.8.self_attention.dense.weight',\n",
       "              tensor([[ 0.0066, -0.0033,  0.0073,  ..., -0.0060, -0.0287,  0.0118],\n",
       "                      [-0.0147, -0.0208, -0.0055,  ..., -0.0176,  0.0274,  0.0303],\n",
       "                      [ 0.0079,  0.0084,  0.0009,  ...,  0.0014, -0.0108, -0.0155],\n",
       "                      ...,\n",
       "                      [-0.0062, -0.0226,  0.0260,  ...,  0.0191, -0.0184,  0.0141],\n",
       "                      [ 0.0449, -0.0113, -0.0175,  ..., -0.0131, -0.0139, -0.0013],\n",
       "                      [ 0.0128, -0.0026,  0.0041,  ..., -0.0077,  0.0068, -0.0008]])),\n",
       "             ('transformer.h.8.self_attention.dense.bias',\n",
       "              tensor([-0.0353, -0.0182, -0.0062,  ..., -0.1779, -0.0287,  0.0010])),\n",
       "             ('transformer.h.8.post_attention_layernorm.weight',\n",
       "              tensor([0.9038, 1.2656, 0.7300,  ..., 1.7080, 1.3701, 0.4604])),\n",
       "             ('transformer.h.8.post_attention_layernorm.bias',\n",
       "              tensor([ 0.0472,  0.0611,  0.0557,  ..., -0.0213,  0.0298, -0.0208])),\n",
       "             ('transformer.h.8.mlp.dense_h_to_4h.weight',\n",
       "              tensor([[ 1.2302e-03, -1.6922e-02,  6.7558e-03,  ..., -1.7271e-03,\n",
       "                       -1.6800e-02,  1.6022e-02],\n",
       "                      [ 7.6485e-03,  1.7532e-02,  8.6355e-04,  ...,  2.7863e-02,\n",
       "                        1.1024e-02,  5.2834e-03],\n",
       "                      [ 1.1200e-02,  8.3237e-03, -2.0401e-02,  ...,  2.4223e-03,\n",
       "                       -1.2268e-02,  6.8283e-04],\n",
       "                      ...,\n",
       "                      [ 2.0844e-02, -2.6566e-02,  3.8872e-03,  ..., -1.1894e-02,\n",
       "                       -9.5444e-03, -1.3161e-02],\n",
       "                      [-4.9530e-02,  2.0676e-02,  5.1498e-03,  ..., -2.8732e-02,\n",
       "                        3.9093e-02, -7.6675e-03],\n",
       "                      [ 8.6308e-05, -2.0027e-03, -3.6713e-02,  ..., -1.5732e-02,\n",
       "                        8.1558e-03,  3.2593e-02]])),\n",
       "             ('transformer.h.8.mlp.dense_h_to_4h.bias',\n",
       "              tensor([-1.6626e-01, -1.8774e-01, -2.2620e-01, -1.1304e-01, -2.0093e-01,\n",
       "                      -4.8877e-01,  3.6285e-02, -3.3417e-02, -3.9111e-01, -5.8643e-01,\n",
       "                      -2.3096e-01, -6.7627e-01, -5.4004e-01, -8.9417e-02, -4.1016e-01,\n",
       "                      -1.5442e-01,  6.6528e-02, -3.7323e-02, -5.9668e-01, -1.1096e-01,\n",
       "                      -8.4473e-02, -1.7590e-01,  1.0339e-01, -2.6440e-01, -1.3802e-02,\n",
       "                      -1.1743e-01, -1.9714e-01, -6.2744e-02, -7.4036e-02,  2.0294e-02,\n",
       "                      -2.9980e-01, -2.2595e-01, -1.6089e-01, -5.2795e-02, -8.5022e-02,\n",
       "                      -9.7778e-02, -2.2034e-01, -6.7688e-02, -1.6235e-01, -8.6136e-03,\n",
       "                      -4.4604e-01, -1.4136e-01, -2.7161e-02, -1.7654e-02,  1.0895e-01,\n",
       "                      -2.6932e-02, -3.9697e-01, -3.2520e-01,  9.5154e-02, -8.8818e-01,\n",
       "                       1.1841e-02, -4.3907e-03, -5.4199e-02, -2.1912e-02, -4.6616e-03,\n",
       "                      -2.6050e-01,  5.1941e-02, -8.9417e-02, -1.7041e-01, -1.0205e-01,\n",
       "                      -2.6270e-01, -5.0244e-01, -4.8096e-01, -1.1896e-01,  2.0859e-02,\n",
       "                      -8.8196e-02, -4.0454e-01, -1.6754e-02, -3.8208e-01, -3.5425e-01,\n",
       "                      -1.1212e-01, -1.5002e-01, -1.4099e-01, -7.2314e-01, -9.6802e-02,\n",
       "                      -6.4355e-01,  1.7365e-02, -2.8271e-01, -3.4692e-01, -1.0980e-01,\n",
       "                       3.0228e-02, -9.0332e-02, -5.2734e-02,  6.7566e-02, -1.7395e-01,\n",
       "                      -1.5674e-01, -3.4497e-01, -5.0049e-01, -6.2158e-01, -2.9480e-02,\n",
       "                      -5.0293e-01, -7.5391e-01, -1.3232e-01, -5.4352e-02, -2.0044e-01,\n",
       "                      -3.0823e-02, -1.2947e-02, -4.2798e-01, -1.5316e-03, -1.0944e-01,\n",
       "                      -1.7102e-01, -2.1729e-01, -4.1431e-01, -1.0956e-01, -2.8247e-01,\n",
       "                       1.5747e-02, -1.8909e-01, -3.7988e-01, -1.3586e-01, -8.4082e-01,\n",
       "                      -2.4536e-01, -4.0918e-01, -3.9459e-02, -5.2295e-01,  2.6718e-02,\n",
       "                      -5.1849e-02,  6.2622e-02, -5.6982e-01, -1.7725e-01,  6.0005e-03,\n",
       "                      -2.4255e-01, -3.7573e-01, -5.6274e-02, -7.0496e-02, -3.8971e-02,\n",
       "                      -3.0045e-02, -6.7322e-02, -6.7078e-02,  1.7593e-02, -2.7100e-02,\n",
       "                       8.5083e-02,  3.4973e-02, -2.7817e-02, -1.3367e-02, -2.1936e-01,\n",
       "                      -1.3214e-02, -1.1578e-01, -5.5908e-02, -4.0381e-01, -5.3497e-02,\n",
       "                      -3.7354e-02, -5.2612e-02, -1.1224e-01, -1.0229e-01, -3.3350e-01,\n",
       "                      -7.5867e-02, -1.6028e-01, -2.3071e-01, -6.0840e-01, -1.8811e-01,\n",
       "                      -5.2429e-02, -2.5269e-02, -1.4417e-01, -1.6150e-01, -4.8706e-01,\n",
       "                      -7.9492e-01, -1.4595e-02, -2.5732e-01, -7.6111e-02, -3.0737e-01,\n",
       "                      -1.9958e-02, -2.6337e-02, -4.7388e-01, -5.1318e-01, -1.5149e-01,\n",
       "                      -1.7792e-02, -2.9388e-02, -2.7905e-01, -8.9172e-02, -2.0046e-03,\n",
       "                      -6.9618e-03, -4.5654e-01, -1.4763e-02,  3.1528e-03, -1.0312e+00,\n",
       "                      -2.8613e-01, -7.8201e-03,  5.5786e-02, -5.2588e-01, -3.6841e-01,\n",
       "                      -2.8369e-01, -3.1348e-01, -6.0608e-02, -1.1334e-01, -5.1178e-02,\n",
       "                      -2.1240e-01, -6.8115e-02,  3.5675e-02,  1.1635e-02, -6.4209e-02,\n",
       "                      -1.7871e-01, -5.2246e-01, -1.4954e-01, -2.1667e-01, -3.2910e-01,\n",
       "                      -3.5303e-01, -3.4009e-01, -8.1711e-03, -6.2439e-02, -5.9326e-02,\n",
       "                      -1.4001e-01, -5.7568e-01, -5.9277e-01, -2.5098e-01, -1.3794e-01,\n",
       "                      -8.1482e-02, -4.7827e-01, -6.2842e-01,  1.8091e-01, -7.4219e-02,\n",
       "                      -2.5604e-02, -3.2642e-01, -5.5145e-02, -4.0796e-01, -6.8848e-02,\n",
       "                      -4.0259e-01,  7.7705e-03, -8.2825e-02, -4.0747e-01, -1.0078e+00,\n",
       "                      -1.5381e-01, -3.0884e-01, -5.2979e-02, -1.6479e-02, -6.9873e-01,\n",
       "                      -2.2107e-01, -5.0342e-01, -5.0812e-02, -4.5190e-01, -5.7959e-01,\n",
       "                      -5.5046e-03, -1.7773e-01,  3.8635e-02, -1.5198e-01, -2.0630e-01,\n",
       "                      -3.8525e-01, -4.3152e-02, -8.0261e-02, -1.5854e-02, -8.9539e-02,\n",
       "                      -2.3206e-01, -5.9418e-02, -3.4009e-01, -6.1890e-02, -2.9419e-02,\n",
       "                      -5.0830e-01, -2.0117e-01, -2.5684e-01, -7.9346e-01, -1.2085e-01,\n",
       "                       3.2288e-02, -1.2408e-01,  4.3762e-02, -6.2164e-02, -3.7402e-01,\n",
       "                      -5.7227e-01, -4.4946e-01, -7.0007e-02,  6.4941e-02,  2.2232e-02,\n",
       "                      -6.2439e-02, -4.8401e-02, -4.3433e-01, -1.6321e-01, -1.1926e-01,\n",
       "                      -9.2896e-02, -5.6494e-01, -8.3105e-01,  5.6419e-03,  1.3557e-02,\n",
       "                      -4.6021e-01,  4.0802e-02, -9.7534e-02, -3.9917e-02, -6.6772e-02,\n",
       "                      -5.8289e-03, -3.3276e-01, -1.3672e-01, -1.3025e-01, -3.8525e-01,\n",
       "                      -3.4961e-01, -7.1631e-01, -9.3201e-02, -2.5562e-01, -4.1211e-01,\n",
       "                      -3.5938e-01, -2.4646e-01, -4.2572e-02, -1.3733e-02, -3.0762e-01,\n",
       "                      -7.6599e-02, -8.1238e-02, -2.0203e-01, -1.1041e-01, -2.5681e-02,\n",
       "                       5.2704e-02, -1.4771e-01, -5.0049e-01, -5.6671e-02, -2.8174e-01,\n",
       "                       7.6721e-02, -7.6709e-01, -5.8301e-01, -9.9243e-02,  5.3650e-02,\n",
       "                      -6.1816e-01, -1.0022e-01, -7.8076e-01, -7.9163e-02, -1.8555e-01,\n",
       "                      -8.9783e-02, -2.8882e-01, -5.9229e-01, -4.4434e-01, -4.7607e-02,\n",
       "                      -2.6337e-02, -5.3772e-02, -4.3518e-02,  6.3599e-02, -2.0349e-01,\n",
       "                      -3.2324e-01,  5.2765e-02, -2.3331e-02, -2.1057e-01, -2.6001e-01,\n",
       "                      -6.9458e-02, -7.2441e-03, -1.4626e-02, -3.0859e-01,  2.8793e-02,\n",
       "                      -1.5161e-01,  2.1988e-02, -1.3586e-01, -1.2169e-02, -3.9337e-02,\n",
       "                      -4.9292e-01, -2.3975e-01, -3.4973e-02, -1.2372e-01, -1.3806e-01,\n",
       "                      -3.4985e-01, -7.8125e-02, -7.0740e-02, -1.8225e-01, -4.0375e-02,\n",
       "                      -4.5654e-01, -2.1362e-01, -4.3884e-02, -4.6948e-01, -1.3721e-01,\n",
       "                      -9.2969e-01, -5.8838e-02,  2.0782e-02, -1.5234e-01, -9.8572e-02,\n",
       "                      -2.1021e-01, -1.6248e-01, -5.8624e-02, -6.8457e-01, -2.2507e-03,\n",
       "                      -1.5160e-02, -1.4233e-01, -6.0693e-01, -8.4763e-03, -2.2888e-02,\n",
       "                      -3.1055e-01, -1.1420e-01, -4.9097e-01, -8.8928e-02, -4.6045e-01,\n",
       "                      -4.5197e-02, -3.0624e-02, -1.8286e-01, -5.7556e-02, -1.4626e-02,\n",
       "                      -8.5498e-01, -2.8732e-02, -2.2797e-02, -2.0117e-01, -8.2764e-02,\n",
       "                      -7.2144e-02, -2.9688e-01, -1.8542e-01, -2.4561e-01,  2.0923e-01,\n",
       "                      -6.5527e-01, -1.1255e-01,  6.6589e-02, -7.1289e-02,  2.4689e-02,\n",
       "                      -2.5928e-01, -1.4404e-01, -7.5378e-02, -1.3354e-01,  1.1707e-01,\n",
       "                      -4.2999e-02, -1.1841e-01, -4.0430e-01, -6.9336e-02, -2.7075e-01,\n",
       "                      -2.0862e-01, -2.0276e-01, -6.1859e-02, -7.0801e-01, -3.2910e-01,\n",
       "                       6.8176e-02, -4.6216e-01, -7.8760e-01, -5.3125e-01, -4.8767e-02,\n",
       "                      -2.6074e-01, -3.3301e-01,  3.2082e-03, -1.7725e-01, -1.0876e-01,\n",
       "                      -3.2471e-01, -7.8491e-02, -1.8591e-01, -4.4678e-01, -1.2558e-02,\n",
       "                      -3.5065e-02, -3.3905e-02, -1.2286e-01, -5.9180e-01, -4.5068e-01,\n",
       "                      -5.6732e-02, -3.1891e-02, -5.8545e-01, -3.1799e-02, -5.4199e-01,\n",
       "                      -2.9932e-01, -4.8364e-01, -6.5820e-01, -3.6646e-01, -2.0667e-01,\n",
       "                      -6.7041e-01, -4.8511e-01, -7.4268e-01, -7.3608e-02, -8.8989e-02,\n",
       "                      -9.3140e-02, -5.5859e-01, -9.8877e-02, -3.0884e-01, -3.2593e-01,\n",
       "                      -1.9739e-01, -1.5485e-04, -6.0449e-01, -2.0056e-01, -1.1267e-01,\n",
       "                      -5.4535e-02, -8.5999e-02, -6.6406e-01, -1.5100e-01, -1.4319e-01,\n",
       "                      -2.8149e-01, -4.7302e-02, -2.0178e-01, -4.2236e-01, -1.3220e-01,\n",
       "                      -5.9814e-02, -4.4098e-02, -3.7476e-01, -6.4636e-02, -6.1523e-01,\n",
       "                      -3.2349e-01, -4.9934e-03, -4.0845e-01, -3.7842e-01, -7.1564e-03,\n",
       "                      -2.7490e-01, -3.0444e-01, -4.4873e-01, -4.1528e-01, -1.5015e-01,\n",
       "                      -6.3477e-02, -4.6844e-02, -1.1218e-01, -2.3804e-02, -6.6650e-02,\n",
       "                      -5.4736e-01, -1.9714e-02, -2.7002e-01, -2.8519e-02, -6.9922e-01,\n",
       "                      -1.8652e-01, -6.9775e-01, -5.5351e-03,  5.0659e-02, -4.2212e-01,\n",
       "                      -7.1106e-02, -3.9185e-01, -4.6692e-02, -2.8091e-02, -1.4172e-01,\n",
       "                      -5.4053e-01, -7.3340e-01, -2.1289e-01, -3.9111e-01, -7.6025e-01,\n",
       "                      -7.2449e-02, -2.1851e-01, -7.2571e-02, -1.5918e-01, -4.9854e-01,\n",
       "                      -2.4902e-02, -3.0350e-02, -7.8674e-02, -6.8298e-02,  6.1096e-02,\n",
       "                      -7.4829e-02,  1.1646e-01,  3.8055e-02, -2.5293e-01, -2.5781e-01,\n",
       "                      -8.6609e-02, -2.1729e-02, -2.4078e-02,  9.9792e-03, -7.8308e-02,\n",
       "                      -1.1676e-01, -3.8849e-02, -1.8384e-01, -7.3853e-02, -6.7139e-02,\n",
       "                      -3.3875e-02, -4.5441e-02, -1.6138e-01, -1.7493e-01, -3.0457e-02,\n",
       "                      -1.1804e-01, -2.9761e-01, -2.9358e-02, -2.0508e-01, -2.8641e-02,\n",
       "                      -3.6279e-01, -6.5308e-02, -2.8345e-01, -1.8164e-01, -3.2043e-02,\n",
       "                      -1.2695e-02, -6.1230e-01, -5.9319e-04,  3.6106e-03, -1.6272e-01,\n",
       "                      -2.9388e-02, -2.0679e-01, -2.5098e-01, -5.8740e-01,  4.6043e-03,\n",
       "                      -2.1167e-01, -4.3304e-02, -4.7534e-01, -2.3254e-01, -6.3477e-01,\n",
       "                      -7.8320e-01, -4.3427e-02, -2.4365e-01, -1.0880e-02, -1.1688e-01,\n",
       "                       1.2924e-02, -3.3447e-01, -1.0309e-01, -2.4133e-01, -1.4209e-01,\n",
       "                      -2.5558e-02, -8.7402e-02, -1.0480e-01, -7.8027e-01, -2.4261e-02,\n",
       "                      -6.9763e-02,  5.7220e-02, -3.6646e-01, -5.7227e-01, -1.3208e-01,\n",
       "                      -6.6992e-01, -3.6328e-01, -5.9717e-01, -5.7812e-01,  3.8971e-02,\n",
       "                      -3.6841e-01, -1.2505e-02, -2.5586e-01, -9.0698e-02, -1.8628e-01,\n",
       "                      -7.3853e-02, -4.2554e-01,  1.9165e-02, -5.0537e-01, -2.0593e-01,\n",
       "                      -2.6221e-01, -4.1040e-01, -1.7480e-01, -1.0597e-02, -3.4424e-01,\n",
       "                      -1.0490e-02, -2.2278e-01, -3.5034e-01, -2.3206e-01, -6.8555e-01,\n",
       "                       4.7852e-02, -4.3701e-01, -1.4612e-01, -1.6983e-02, -3.3691e-02,\n",
       "                      -3.9575e-01, -5.4150e-01,  3.7598e-02, -6.4331e-02,  1.9180e-02,\n",
       "                       2.7122e-03, -2.0027e-03, -2.7173e-01, -2.9980e-01,  1.8188e-02,\n",
       "                      -5.0537e-01, -1.1185e-02, -1.7303e-02, -6.0156e-01, -9.2773e-02,\n",
       "                       2.4933e-02, -1.5503e-01, -1.6650e-01, -4.4653e-01, -5.0079e-02,\n",
       "                      -8.4900e-02,  1.2596e-02, -1.6541e-01, -1.1124e-02, -7.6465e-01,\n",
       "                      -5.4541e-01, -1.9302e-02, -1.6296e-01, -7.4463e-02, -4.0552e-01,\n",
       "                      -6.7725e-01, -1.0681e-01, -5.4297e-01, -1.3275e-02, -3.9478e-01,\n",
       "                      -1.4111e-01, -1.6345e-01, -2.1460e-01, -1.5881e-01, -2.9053e-02,\n",
       "                      -5.2307e-02, -2.9297e-01, -2.6352e-02, -7.5073e-02, -7.0166e-01,\n",
       "                      -2.3901e-01, -2.6489e-01, -6.5613e-02, -1.1951e-01, -2.3376e-01,\n",
       "                      -2.4719e-01, -1.6418e-02, -5.1361e-02, -3.2495e-01, -2.6901e-02,\n",
       "                      -6.1328e-01, -2.0715e-01, -4.2175e-02,  2.3108e-01, -2.0056e-01,\n",
       "                       5.6244e-02,  2.2522e-02,  3.0441e-02, -7.4402e-02, -1.9958e-01,\n",
       "                       4.3549e-02, -2.0676e-02, -3.5107e-01, -3.6938e-01, -1.4221e-01,\n",
       "                      -4.4220e-02, -1.0602e-01, -3.1738e-01, -3.4766e-01, -6.1865e-01,\n",
       "                      -2.8906e-01, -1.3268e-02, -7.4219e-02,  8.7128e-03, -1.0870e-01,\n",
       "                      -9.2773e-02, -7.6355e-02, -6.4844e-01, -3.3142e-02, -1.0669e-01,\n",
       "                      -3.8257e-01, -1.6016e-01, -1.2329e-01, -2.9980e-01, -5.1270e-01,\n",
       "                      -6.0400e-01, -4.0619e-02, -7.7881e-02, -1.1005e-01, -2.1683e-02,\n",
       "                      -6.0333e-02, -1.5845e-01, -1.5186e-01, -1.5283e-01, -4.2456e-01,\n",
       "                      -8.5083e-02, -4.2627e-01, -7.1533e-02, -7.3730e-02,  4.7821e-02,\n",
       "                      -2.7905e-01, -1.3354e-01, -1.5515e-01, -6.1963e-01, -4.9878e-01,\n",
       "                      -1.7441e-02, -1.3110e-01, -1.8445e-01, -1.6527e-03, -6.6711e-02,\n",
       "                      -4.6997e-02, -2.8003e-01, -3.9404e-01, -1.9568e-01, -3.0957e-01,\n",
       "                      -1.0913e-01, -4.1528e-01, -1.9501e-02,  1.0735e-02, -6.4697e-02,\n",
       "                      -1.6748e-01, -6.4392e-02, -7.4561e-01, -7.4805e-01, -4.4873e-01,\n",
       "                      -6.0596e-01, -3.5327e-01, -6.4148e-02, -5.7666e-01, -6.9141e-01,\n",
       "                      -3.7109e-01, -1.2054e-01, -1.5234e-01, -1.8005e-01, -8.8989e-02,\n",
       "                      -1.2158e-01, -4.4739e-02, -6.3135e-01, -6.9946e-02, -9.4971e-02,\n",
       "                      -1.8997e-02, -5.7275e-01, -4.8853e-01, -1.7249e-01, -1.1218e-01,\n",
       "                       6.5979e-02, -1.2756e-02, -1.4539e-01, -3.0365e-02, -9.8572e-02,\n",
       "                      -3.2684e-02, -5.4980e-01, -2.0398e-01, -1.1040e-02, -1.4557e-02,\n",
       "                       2.4750e-02, -4.8126e-02, -4.5654e-01, -4.3018e-01,  2.0950e-02,\n",
       "                      -1.0663e-01, -4.5654e-02, -5.8936e-01, -1.3525e-01, -4.1992e-01,\n",
       "                      -1.7761e-01, -1.0040e-01,  1.7502e-02, -1.4978e-01, -3.5083e-01,\n",
       "                      -4.0942e-01, -4.3188e-01, -8.1055e-02, -2.2934e-02,  5.3467e-02,\n",
       "                      -2.9785e-01, -2.5415e-01, -6.7627e-01,  5.6446e-05, -8.8959e-03,\n",
       "                      -4.1748e-01, -8.0859e-01,  8.8318e-02, -5.5908e-02, -4.5166e-01,\n",
       "                      -4.7656e-01, -2.1622e-02, -3.7354e-02, -5.3613e-01, -5.7324e-01,\n",
       "                      -6.5491e-02, -7.5781e-01, -3.8110e-01])),\n",
       "             ('transformer.h.8.mlp.dense_4h_to_h.weight',\n",
       "              tensor([[ 0.0044, -0.0224,  0.0051,  ..., -0.0136,  0.0047, -0.0145],\n",
       "                      [ 0.0301, -0.0052,  0.0152,  ...,  0.0370,  0.0171,  0.0002],\n",
       "                      [ 0.0176,  0.0035, -0.0069,  ...,  0.0184,  0.0167,  0.0133],\n",
       "                      ...,\n",
       "                      [-0.0059, -0.0041, -0.0121,  ...,  0.0017, -0.0895, -0.0248],\n",
       "                      [ 0.0159,  0.0018, -0.0089,  ...,  0.0191, -0.0520, -0.0023],\n",
       "                      [ 0.0019, -0.0042, -0.0002,  ..., -0.0043,  0.0206,  0.0074]])),\n",
       "             ('transformer.h.8.mlp.dense_4h_to_h.bias',\n",
       "              tensor([ 0.0567,  0.0073, -0.0007,  ..., -0.0843,  0.0322, -0.0098])),\n",
       "             ('transformer.h.9.input_layernorm.weight',\n",
       "              tensor([1.1260, 1.7012, 1.0039,  ..., 2.2461, 1.7588, 0.8037])),\n",
       "             ('transformer.h.9.input_layernorm.bias',\n",
       "              tensor([ 0.0162,  0.0440,  0.0877,  ...,  0.0591,  0.0028, -0.0005])),\n",
       "             ('transformer.h.9.self_attention.query_key_value.weight',\n",
       "              tensor([[-0.0312, -0.0029,  0.0565,  ...,  0.0012, -0.0245,  0.0582],\n",
       "                      [ 0.0077,  0.0043, -0.0423,  ...,  0.0462,  0.0014,  0.0016],\n",
       "                      [-0.0238,  0.0275, -0.0239,  ..., -0.0286, -0.0340, -0.0059],\n",
       "                      ...,\n",
       "                      [-0.0101, -0.0504,  0.0154,  ..., -0.0172,  0.0020, -0.0038],\n",
       "                      [-0.0126,  0.0169,  0.0181,  ..., -0.0328,  0.0324,  0.0269],\n",
       "                      [-0.0331,  0.0096,  0.0052,  ..., -0.0285,  0.0089,  0.0071]])),\n",
       "             ('transformer.h.9.self_attention.query_key_value.bias',\n",
       "              tensor([-2.7832e-02, -2.0767e-02,  7.6532e-04, -4.6112e-02, -4.8828e-02,\n",
       "                       1.4183e-02, -4.8492e-02, -4.7791e-02, -9.3567e-02,  6.5125e-02,\n",
       "                       2.0325e-01, -7.7515e-02, -2.5879e-02,  7.7515e-02, -1.2347e-01,\n",
       "                       6.7627e-02,  4.6387e-02,  2.0172e-02, -1.1238e-02, -6.1188e-02,\n",
       "                      -3.1708e-02,  4.5197e-02, -3.2410e-02,  2.8946e-02, -1.4114e-02,\n",
       "                      -4.7638e-02,  3.1036e-02,  2.3270e-02,  3.8872e-03, -6.9519e-02,\n",
       "                       1.9348e-02, -3.7415e-02,  4.7943e-02, -1.4023e-02, -1.2215e-02,\n",
       "                      -4.4861e-02,  5.1086e-02, -1.2520e-02,  1.0999e-01, -3.7964e-02,\n",
       "                       1.3184e-01,  1.6101e-01,  2.9312e-02, -5.9052e-02,  1.3281e-01,\n",
       "                       3.2684e-02, -2.5708e-01,  4.9042e-02, -7.4951e-02, -7.1144e-03,\n",
       "                       5.5981e-04,  3.1006e-01,  7.1533e-01, -9.4727e-02,  2.7734e-01,\n",
       "                       7.0007e-02, -3.4119e-02,  6.8016e-03, -3.3997e-02, -7.9590e-02,\n",
       "                       5.8899e-03, -4.1309e-01, -7.0374e-02,  4.0100e-02, -4.2969e-02,\n",
       "                      -1.7105e-02, -1.7847e-01,  1.3916e-01,  3.3447e-02,  8.5266e-02,\n",
       "                       2.1133e-02, -4.2076e-03, -2.5488e-01, -1.8457e-01, -5.2734e-01,\n",
       "                       1.8933e-01, -1.8701e-01,  1.3953e-01,  2.6489e-02,  2.7725e-02,\n",
       "                       7.0312e-02, -2.4854e-01, -1.3489e-01,  5.6396e-01,  3.8989e-01,\n",
       "                      -2.5439e-01,  2.2949e+00, -5.2246e-02,  1.5784e-01, -1.1993e-01,\n",
       "                       4.7089e-02,  3.7744e-01, -7.6294e-02,  1.8921e-01,  3.4332e-02,\n",
       "                      -2.2388e-01, -5.4474e-02, -7.9651e-02, -2.0813e-01, -1.0522e-01,\n",
       "                       3.0737e-01, -2.8979e-01, -6.1310e-02,  1.6113e-01, -1.5710e-01,\n",
       "                       9.3872e-02,  8.5571e-02, -1.5430e-01,  9.4543e-02, -1.0718e-01,\n",
       "                       1.0323e-02,  1.2805e-01,  2.3853e-01,  2.0435e-01,  4.5459e-01,\n",
       "                       1.3708e-01,  1.2756e-01,  7.4341e-02, -3.0689e-03, -2.4048e-02,\n",
       "                      -6.1310e-02, -6.1768e-02, -2.1484e-01, -2.1008e-01,  1.2408e-01,\n",
       "                      -8.8318e-02,  5.7098e-02, -2.1301e-02,  6.8604e-02,  5.1758e-02,\n",
       "                      -2.0035e-02,  1.8326e-02,  8.9661e-02,  6.6467e-02, -5.3131e-02,\n",
       "                      -1.5247e-01, -1.7529e-01, -8.7952e-02,  2.3285e-02, -3.9948e-02,\n",
       "                      -4.0955e-02, -4.7241e-02,  4.3144e-03,  5.4016e-02, -1.0712e-01,\n",
       "                      -5.4779e-02,  1.7517e-01, -3.2410e-02, -4.9316e-02,  8.9111e-02,\n",
       "                       1.6309e+00,  4.4067e-02,  1.1768e-01, -3.1143e-02,  8.9844e-02,\n",
       "                      -3.4058e-02, -3.0563e-02,  7.5378e-02, -5.4131e-03,  2.4780e-02,\n",
       "                      -1.1194e-01,  3.9764e-02,  1.2743e-04, -1.3371e-03,  2.2797e-02,\n",
       "                      -1.0506e-02, -1.0022e-01, -3.5370e-02, -6.4163e-03, -1.0382e-01,\n",
       "                      -1.2589e-02,  9.4910e-02,  6.3599e-02, -1.4832e-02,  6.4514e-02,\n",
       "                      -4.0375e-02, -4.8859e-02,  2.1835e-02,  2.7856e-01,  9.8145e-02,\n",
       "                       1.6260e-01, -1.0272e-01,  1.8377e-03,  7.3975e-02,  8.6060e-02,\n",
       "                       7.9956e-02,  1.1055e-02,  2.7832e-02,  5.8838e-02,  1.0919e-01,\n",
       "                       1.6830e-02,  5.7556e-02,  9.7412e-02,  2.8885e-02, -6.3965e-02,\n",
       "                       6.1963e-01, -1.2445e-01,  4.6173e-02,  3.4229e-01, -3.2202e-01,\n",
       "                       7.8506e-03, -1.5979e-01,  1.3367e-01,  1.2140e-01,  2.3060e-03,\n",
       "                      -2.7197e-01,  1.1414e-01,  2.7246e-01, -1.1035e-01,  3.3545e-01,\n",
       "                      -1.4685e-01, -4.7211e-02, -2.0840e+00,  9.0210e-02,  2.7490e-01,\n",
       "                      -2.5049e-01,  2.8223e-01,  6.8909e-02,  1.1635e-02, -1.4160e-01,\n",
       "                      -1.2585e-01, -3.5498e-01,  4.7974e-02,  3.0176e-01,  4.3018e-01,\n",
       "                       2.2302e-01,  8.3008e-02,  4.1748e-02, -3.3984e-01,  1.5356e-01,\n",
       "                       1.4307e-01, -8.4534e-02,  9.3628e-02, -1.0449e-01,  9.0759e-02,\n",
       "                      -3.2275e-01, -3.9307e-01,  9.9365e-02, -9.7412e-02,  7.0190e-02,\n",
       "                       3.3997e-02,  1.8188e-02,  1.6479e-02,  7.4463e-02, -2.3694e-01,\n",
       "                      -4.2999e-02,  1.4282e-01,  9.2087e-03, -1.3281e+00,  1.4893e-01,\n",
       "                       4.3976e-02, -1.0370e-01,  6.7688e-02,  4.4946e-01, -1.3452e-01,\n",
       "                      -2.4216e-02, -1.1238e-02, -1.2352e-02, -7.2575e-04,  7.1793e-03,\n",
       "                       3.4821e-02,  1.6373e-02,  5.4131e-03, -2.7130e-02, -2.8168e-02,\n",
       "                      -3.0167e-02,  5.6124e-04, -6.3705e-03, -3.3936e-02,  2.0599e-02,\n",
       "                      -3.5126e-02,  2.3849e-02, -7.0610e-03,  3.8055e-02, -4.8126e-02,\n",
       "                       5.6335e-02,  1.8921e-02,  4.9896e-02, -2.8801e-03, -2.2049e-02,\n",
       "                      -1.9165e-02, -3.8376e-03, -8.5983e-03, -4.0833e-02, -1.3676e-03,\n",
       "                       3.4698e-02,  3.0670e-03,  2.1973e-02, -1.1681e-02,  1.4053e-02,\n",
       "                      -5.2786e-04, -1.0826e-02, -4.3945e-02, -2.3514e-02, -2.2400e-02,\n",
       "                      -1.2268e-02, -8.9264e-03, -1.2306e-02, -2.2369e-02, -2.5650e-02,\n",
       "                       2.6741e-03,  1.8234e-02,  1.8265e-02,  8.4686e-03, -2.5253e-02,\n",
       "                       3.4882e-02,  3.3325e-02, -2.4216e-02, -6.6772e-02, -3.2074e-02,\n",
       "                      -1.1368e-03, -3.6926e-02,  1.4435e-02,  2.3422e-02, -6.8176e-02,\n",
       "                       2.6428e-02, -4.2381e-03,  5.4657e-02, -2.2720e-02, -5.5466e-03,\n",
       "                       3.5859e+00, -9.8682e-01, -1.9688e+00, -5.2588e-01,  1.7246e+00,\n",
       "                      -2.5176e+00,  3.0469e+00, -2.4570e+00,  3.1689e-01,  2.2168e+00,\n",
       "                      -2.4121e+00,  2.9316e+00,  2.7617e+00, -2.1172e+00, -3.1504e+00,\n",
       "                       3.7129e+00,  9.6387e-01,  4.4062e+00,  4.3008e+00,  1.8389e+00,\n",
       "                      -5.5566e-01,  2.1797e+00,  2.3750e+00, -2.1445e+00,  1.0125e+01,\n",
       "                       2.4668e+00,  1.4912e+00, -3.3027e+00,  1.9580e+00,  2.8887e+00,\n",
       "                       3.4102e+00, -1.2178e+00,  3.2305e+00,  5.9692e-02, -1.5781e+00,\n",
       "                       1.6660e+00, -4.8035e-02,  5.4102e-01,  1.4150e+00, -1.9287e+00,\n",
       "                       7.8008e+00, -4.4062e+00, -2.4082e+00,  2.7988e+00,  2.3008e+00,\n",
       "                       3.8730e+00,  4.0352e+00, -6.0059e-01,  3.3184e+00,  5.6343e-03,\n",
       "                      -1.7891e+00, -2.9766e+00,  1.4453e+00,  3.2578e+00, -2.7852e+00,\n",
       "                      -3.4473e+00,  6.6504e-01,  1.4570e+00, -3.8145e+00, -5.9143e-02,\n",
       "                      -8.5078e+00, -2.4375e+00,  1.0400e+00,  1.6123e+00, -5.3253e-03,\n",
       "                      -4.2285e-01, -2.4338e-03,  2.3010e-02, -8.3466e-03, -5.9586e-03,\n",
       "                       7.7667e-03, -1.4944e-03, -1.7052e-03, -1.8311e-02,  8.1100e-03,\n",
       "                      -3.8239e-02,  5.7983e-03, -9.5978e-03,  4.3091e-02, -1.8250e-02,\n",
       "                      -2.3315e-02, -1.0857e-02,  4.5776e-03,  2.2598e-02,  6.4468e-03,\n",
       "                      -1.6541e-02,  9.6283e-03, -1.7624e-02,  5.8403e-03, -2.3727e-03,\n",
       "                       1.0330e-02,  4.1748e-02,  2.9785e-02, -8.8196e-03, -1.9318e-02,\n",
       "                       1.5205e-02, -2.9327e-02, -1.7090e-02, -7.7286e-03,  5.5573e-02,\n",
       "                       1.6565e-03, -1.6022e-02,  2.8744e-03, -1.2985e-02,  4.1779e-02,\n",
       "                      -2.4887e-02,  2.2003e-02,  6.3965e-02, -1.0071e-01, -1.8570e-02,\n",
       "                      -1.2718e-02,  8.3496e-02,  1.5244e-02, -5.5847e-03, -2.2797e-02,\n",
       "                      -1.6113e-02, -1.4252e-02,  2.6188e-03, -1.0452e-02,  3.9429e-02,\n",
       "                       1.8982e-02,  2.3254e-02, -4.9057e-03, -1.0750e-02,  9.6741e-03,\n",
       "                       2.5467e-02, -3.9795e-02,  1.8524e-02,  7.4316e-01,  2.4043e+00,\n",
       "                      -3.0640e-01, -4.8125e+00, -3.8203e+00, -1.7578e+00, -2.9922e+00,\n",
       "                      -1.8496e+00, -1.3599e-01,  5.0342e-01, -4.7607e-01,  9.4971e-01,\n",
       "                      -2.9785e+00, -1.3174e+00, -2.6680e+00,  4.4434e-01, -3.0508e+00,\n",
       "                      -3.0645e+00, -3.8047e+00,  4.3516e+00, -2.3672e+00,  3.2090e+00,\n",
       "                       3.1172e+00,  1.6338e+00,  1.5049e+00,  5.0938e+00, -1.5791e+00,\n",
       "                      -3.2593e-01,  4.1094e+00, -8.0615e-01,  9.8291e-01, -3.1484e+00,\n",
       "                       6.5155e-03, -3.3081e-01,  2.2441e+00,  4.0078e+00, -2.1680e-01,\n",
       "                       1.0225e+00,  6.9531e-01,  1.3809e-03, -3.0371e+00, -3.3618e-01,\n",
       "                       2.1094e+00, -1.4121e+00, -1.8643e+00,  1.9072e+00,  4.9727e+00,\n",
       "                      -7.1240e-01, -4.9883e+00, -1.4526e-01,  2.4341e-01, -4.2407e-01,\n",
       "                       3.4277e+00,  4.6055e+00, -2.0391e+00, -5.7109e+00, -4.1943e-01,\n",
       "                      -1.2314e+00, -1.9785e+00,  2.0977e+00, -2.1875e+00, -1.2461e+00,\n",
       "                       4.9258e+00,  1.4678e+00, -7.0801e-01,  1.2732e-01, -9.4070e-03,\n",
       "                      -6.1523e-01,  3.0542e-01,  3.1226e-01, -3.6621e-01,  2.1033e-01,\n",
       "                      -4.5068e-01,  1.8604e-01,  3.9185e-01, -7.6965e-02,  3.7915e-01,\n",
       "                      -2.9102e-01,  3.0957e-01,  5.4535e-02, -1.5833e-01, -5.0635e-01,\n",
       "                       6.6064e-01,  3.1421e-01,  2.6001e-02, -2.8247e-01, -7.4170e-01,\n",
       "                      -2.3987e-01, -1.4807e-01,  6.1719e-01, -2.2705e-01,  5.4492e-01,\n",
       "                       5.8887e-01,  7.0020e-01, -2.7441e-01,  1.3940e-01,  4.6680e-01,\n",
       "                       4.7144e-01, -1.0962e-01,  5.9863e-01,  4.3286e-01,  6.5039e-01,\n",
       "                      -2.1362e-01, -2.7515e-01,  2.5928e-01, -6.9031e-02, -3.0811e-01,\n",
       "                      -2.1777e+00, -4.4336e-01,  2.4438e-01, -5.7178e-01,  3.8719e-03,\n",
       "                      -2.1033e-01, -9.1992e-01,  3.0566e-01,  1.7932e-01,  2.0547e+00,\n",
       "                      -2.4673e-02, -7.3633e-01,  3.0811e-01,  2.1838e-01,  2.3633e+00,\n",
       "                      -1.7004e-01,  3.6011e-01, -4.4727e-01,  9.9268e-01,  2.0215e+00,\n",
       "                       2.6758e-01, -2.5665e-02, -4.2648e-03,  1.5465e-02,  4.0741e-02,\n",
       "                      -4.0466e-02,  6.7520e-03,  4.9667e-03,  1.4870e-02, -1.7456e-02,\n",
       "                       7.3013e-03, -9.2392e-03,  6.4468e-03,  3.6163e-02,  3.6049e-03,\n",
       "                       6.5765e-03, -5.9280e-03,  1.3786e-02, -2.5101e-02, -3.7781e-02,\n",
       "                       1.5518e-02,  8.3237e-03,  8.5678e-03,  2.0477e-02, -1.0612e-02,\n",
       "                       3.0167e-02, -3.9459e-02, -3.3295e-02, -2.4551e-02, -1.7731e-02,\n",
       "                      -2.8057e-03, -3.9787e-03,  1.5617e-02,  4.5967e-03,  4.6729e-01,\n",
       "                       2.6749e-02,  1.3695e-02,  3.6865e-02,  9.9468e-04,  2.1683e-02,\n",
       "                       3.6602e-03,  4.8327e-04, -3.3234e-02,  3.8696e-02,  7.2517e-03,\n",
       "                       5.0751e-02,  1.7960e-02,  8.2397e-03, -1.7975e-02, -2.9922e-02,\n",
       "                       2.9770e-02, -5.5351e-03, -6.6414e-03, -1.7822e-02,  1.1612e-02,\n",
       "                       3.9642e-02, -1.5007e-02, -3.0308e-03,  3.1204e-02,  1.1368e-03,\n",
       "                      -4.0169e-03, -6.4125e-03, -6.4392e-03, -2.6941e-04,  3.1219e-02,\n",
       "                      -3.9673e-01, -7.6477e-02,  1.6760e-01, -2.9373e-02, -1.2910e+00,\n",
       "                       1.9006e-01, -2.4011e-01,  4.9988e-02, -1.8860e-01, -6.2500e-01,\n",
       "                      -1.1737e-01, -8.2153e-02, -3.8647e-01, -4.9286e-02, -1.8079e-01,\n",
       "                       5.2588e-01,  1.8677e-01,  1.6406e-01,  1.3330e-01, -1.9812e-01,\n",
       "                      -4.4922e-01, -3.2642e-01,  5.7190e-02, -2.4500e-01,  7.9224e-02,\n",
       "                       5.1367e-01, -2.5574e-02, -6.8018e-01, -8.8745e-02,  7.8076e-01,\n",
       "                      -2.4194e-01,  4.2773e-01, -1.6003e-01,  2.5977e-01, -9.4910e-02,\n",
       "                      -1.2524e-01,  3.0195e+00,  5.3662e-01,  6.9971e-01,  1.0370e-01,\n",
       "                       2.4731e-01, -5.9277e-01,  2.3206e-01, -1.0614e-01,  1.4014e-01,\n",
       "                      -1.5442e-02,  3.1177e-01,  3.2349e-01, -3.9038e-01, -1.4478e-01,\n",
       "                      -1.4893e-01, -5.8740e-01,  2.8613e-01,  4.7266e-01, -6.6504e-01,\n",
       "                      -2.0508e-01,  1.0225e+00, -1.0468e-02, -3.7933e-02, -8.5352e-01,\n",
       "                      -4.7290e-01, -6.7969e-01,  5.0415e-02,  2.3999e-01, -2.4414e-02,\n",
       "                      -5.5809e-03, -3.4088e-02,  2.4796e-02,  5.0018e-02, -1.3313e-02,\n",
       "                       3.0975e-02,  5.9052e-03, -5.1666e-02, -1.9714e-02,  3.0411e-02,\n",
       "                      -4.3854e-02,  3.3752e-02,  2.7893e-02,  4.1046e-03, -5.3558e-03,\n",
       "                      -1.2001e-02, -1.5480e-02,  1.7586e-03, -2.2003e-02, -8.0643e-03,\n",
       "                      -1.2329e-02, -3.0243e-02, -1.6678e-02,  5.3375e-02, -2.4765e-02,\n",
       "                       1.4236e-02,  3.6743e-02,  1.7410e-02, -4.9225e-02,  4.5959e-02,\n",
       "                      -3.9734e-02, -1.6586e-02, -5.1849e-02, -3.1128e-02,  2.2217e-02,\n",
       "                       1.8066e-02, -1.5282e-02,  3.2379e-02, -5.6572e-03, -1.1108e-02,\n",
       "                      -2.3270e-02,  2.4490e-02,  4.0321e-03, -5.7739e-02, -3.7109e-02,\n",
       "                       4.1695e-03,  9.0408e-03,  5.8899e-02, -3.1525e-02, -2.3804e-02,\n",
       "                       2.3575e-02, -6.2347e-02,  3.7781e-02,  1.0529e-02, -2.9716e-03,\n",
       "                       6.5842e-03,  8.0414e-03,  3.4668e-02, -3.8300e-02, -1.9836e-02,\n",
       "                       1.9760e-02, -2.7924e-02,  3.5889e-02])),\n",
       "             ('transformer.h.9.self_attention.dense.weight',\n",
       "              tensor([[ 0.0071,  0.0183, -0.0097,  ...,  0.0076,  0.0291,  0.0040],\n",
       "                      [ 0.0012,  0.0026,  0.0028,  ...,  0.0306, -0.0260, -0.0055],\n",
       "                      [ 0.0136,  0.0099,  0.0258,  ...,  0.0002, -0.0063,  0.0025],\n",
       "                      ...,\n",
       "                      [-0.0063, -0.0099, -0.0062,  ...,  0.0210, -0.0024, -0.0082],\n",
       "                      [-0.0171, -0.0029, -0.0186,  ...,  0.0541, -0.0125, -0.0159],\n",
       "                      [ 0.0077,  0.0096, -0.0073,  ..., -0.0038,  0.0069, -0.0161]])),\n",
       "             ('transformer.h.9.self_attention.dense.bias',\n",
       "              tensor([ 0.0028,  0.0027,  0.0206,  ..., -0.0475,  0.0001,  0.0118])),\n",
       "             ('transformer.h.9.post_attention_layernorm.weight',\n",
       "              tensor([0.7954, 1.1738, 0.6509,  ..., 1.6514, 1.3525, 0.4316])),\n",
       "             ('transformer.h.9.post_attention_layernorm.bias',\n",
       "              tensor([ 0.0188,  0.0656,  0.0300,  ...,  0.0049,  0.0120, -0.0161])),\n",
       "             ('transformer.h.9.mlp.dense_h_to_4h.weight',\n",
       "              tensor([[ 0.0421,  0.0054, -0.0282,  ...,  0.0186,  0.0184, -0.0213],\n",
       "                      [-0.0027, -0.0144,  0.0191,  ...,  0.0196,  0.0004, -0.0316],\n",
       "                      [ 0.0037,  0.0137,  0.0113,  ...,  0.0042,  0.0076,  0.0158],\n",
       "                      ...,\n",
       "                      [-0.0189,  0.0206, -0.0051,  ...,  0.0130,  0.0046,  0.0157],\n",
       "                      [-0.0184, -0.0059, -0.0012,  ..., -0.0094, -0.0101,  0.0226],\n",
       "                      [-0.0660,  0.0434,  0.0012,  ...,  0.0079,  0.0037, -0.0055]])),\n",
       "             ('transformer.h.9.mlp.dense_h_to_4h.bias',\n",
       "              tensor([-1.9739e-01, -6.8787e-02, -2.1179e-01, -6.3843e-02, -6.1615e-02,\n",
       "                       9.1309e-02, -1.0907e-01, -4.7656e-01, -3.5217e-02, -1.2634e-02,\n",
       "                      -3.5376e-01,  5.4962e-02, -1.4563e-01, -2.4133e-01, -2.4063e-02,\n",
       "                      -2.1240e-01, -3.3301e-01, -2.2083e-01, -1.1517e-01, -6.5283e-01,\n",
       "                      -3.2861e-01,  9.1309e-02, -3.0615e-01, -1.7075e-02, -4.0991e-01,\n",
       "                       3.9062e-02, -2.9102e-01, -5.4102e-01,  1.5121e-02, -2.9956e-01,\n",
       "                      -3.8940e-02, -3.2715e-02, -5.8496e-01, -6.3672e-01, -7.4805e-01,\n",
       "                       1.1749e-02, -9.8755e-02, -1.7810e-01, -2.8809e-01, -6.7505e-02,\n",
       "                       1.2769e-01,  1.2524e-01, -8.7219e-02, -1.7334e-01, -2.7734e-01,\n",
       "                       2.1225e-02, -6.8604e-02, -9.3002e-03,  8.7646e-02, -8.5693e-02,\n",
       "                      -5.2832e-01, -8.0566e-02, -2.7075e-01, -1.1456e-01, -2.4927e-01,\n",
       "                      -2.9587e-02, -4.8431e-02, -8.1982e-01, -1.5881e-01, -1.8738e-01,\n",
       "                      -4.2419e-02, -6.3782e-02, -6.1377e-01, -9.2529e-01, -6.8481e-02,\n",
       "                      -1.7746e-02, -1.7969e-01, -3.6401e-01, -3.1665e-01, -4.8071e-01,\n",
       "                       9.7534e-02, -8.2520e-02,  1.9189e-01, -2.0081e-02, -4.1748e-01,\n",
       "                      -1.4404e-01, -4.2212e-01, -6.3672e-01, -3.5742e-01, -4.4373e-02,\n",
       "                      -1.1438e-01, -7.1631e-01, -1.8262e-01, -1.3953e-01, -1.8311e-02,\n",
       "                      -1.0480e-01, -6.5674e-01,  1.7242e-02, -1.5735e-01, -2.1387e-01,\n",
       "                      -5.8563e-02,  3.6774e-02,  4.0747e-01, -7.2363e-01, -4.9707e-01,\n",
       "                       1.2830e-01, -3.1787e-01,  8.1711e-03,  4.1008e-03, -5.8398e-01,\n",
       "                      -6.3428e-01, -1.8127e-01, -7.0984e-02, -3.9864e-03, -1.2988e-01,\n",
       "                      -2.2217e-01, -2.0068e-01, -8.1238e-02, -5.4199e-01, -1.0583e-01,\n",
       "                      -3.3521e-01, -1.3513e-01, -1.9641e-01, -1.2510e+00, -2.2241e-01,\n",
       "                       7.8552e-02, -9.7290e-02, -2.0789e-01, -1.7441e-02,  6.8604e-02,\n",
       "                      -5.2795e-02, -1.8604e-01,  2.1713e-02,  3.4760e-02,  1.2769e-01,\n",
       "                      -7.4854e-01, -1.1359e-01, -2.8412e-02,  5.9357e-03, -2.4124e-02,\n",
       "                       7.5256e-02, -4.8981e-03,  6.9824e-02, -1.3269e-01, -1.1475e-01,\n",
       "                      -9.7351e-02, -6.7627e-02, -3.2544e-01, -4.6948e-01, -2.0279e-02,\n",
       "                      -9.4421e-02, -2.1558e-01, -2.1179e-01, -4.1089e-01,  1.4355e-01,\n",
       "                      -2.7930e-01, -4.2633e-02, -6.0107e-01, -2.6001e-01, -5.6299e-01,\n",
       "                      -5.4962e-02, -2.7905e-01, -1.5259e-01,  2.7954e-01, -4.6112e-02,\n",
       "                      -3.5156e-01, -6.0303e-02,  4.5624e-02, -7.4854e-01, -4.0259e-01,\n",
       "                      -3.3112e-02, -2.4048e-02,  2.2620e-01, -2.2058e-01, -4.8877e-01,\n",
       "                      -9.1736e-02,  1.5793e-02, -1.6754e-02,  9.4788e-02, -4.7070e-01,\n",
       "                      -4.7852e-01, -1.4490e-01, -5.7666e-01, -8.5986e-01, -1.2695e-02,\n",
       "                      -1.5222e-01, -4.1779e-02,  3.0884e-01, -2.5467e-02, -2.9199e-01,\n",
       "                       1.9836e-01, -4.5215e-01, -1.1377e-01,  6.2683e-02, -4.9292e-01,\n",
       "                      -1.8237e-01, -2.1558e-01,  8.1543e-02,  2.9388e-02, -1.7615e-01,\n",
       "                      -8.9294e-02, -3.1958e-01,  3.2043e-02, -2.2998e-01, -8.2214e-02,\n",
       "                      -1.7136e-02, -5.5786e-02, -1.8835e-01, -4.2633e-02, -3.8361e-02,\n",
       "                      -6.8054e-02, -8.7097e-02, -3.8391e-02, -1.8408e-01, -5.3310e-04,\n",
       "                      -2.1896e-02, -3.3423e-01, -1.5381e-01, -4.3457e-01, -3.4943e-02,\n",
       "                      -1.5388e-02,  5.8685e-02, -4.1412e-02, -1.7471e-03, -4.4067e-01,\n",
       "                      -1.1463e-03, -7.6953e-01, -1.6406e-01, -1.8726e-01, -2.2949e-02,\n",
       "                      -5.5634e-02, -5.9473e-01, -9.5703e-02, -5.7031e-01, -6.8164e-01,\n",
       "                       4.3518e-02, -5.0720e-02, -3.0396e-02,  1.2952e-01, -2.2705e-01,\n",
       "                      -3.8501e-01, -7.2119e-01, -3.2623e-02, -1.7261e-01, -8.1665e-02,\n",
       "                      -8.4717e-02, -4.1577e-01, -1.5587e-02, -4.8560e-01, -1.0735e-02,\n",
       "                      -1.2405e-02,  2.5131e-02,  2.5284e-02, -2.9736e-01, -9.2468e-03,\n",
       "                      -8.2886e-02, -1.4941e-01, -2.1216e-01, -1.5198e-01,  3.3783e-02,\n",
       "                       1.9775e-02,  7.7087e-02, -2.5098e-01,  8.7891e-03, -1.0126e-01,\n",
       "                      -3.3722e-02, -8.3618e-02, -9.3140e-02, -9.1431e-02, -1.8005e-01,\n",
       "                      -1.0687e-01, -1.2830e-01, -1.8738e-02, -2.2168e-01, -2.6367e-01,\n",
       "                      -3.8361e-02, -1.0736e-01,  2.6283e-03, -3.3057e-01,  7.3051e-03,\n",
       "                      -1.4807e-01,  1.0938e-01, -1.6907e-02, -1.7151e-01, -1.0297e-01,\n",
       "                      -2.6337e-02, -4.1870e-01, -1.7624e-02, -4.8431e-02, -1.7078e-01,\n",
       "                      -4.2847e-01, -9.6817e-03,  4.8889e-02, -2.0447e-01, -2.7710e-01,\n",
       "                      -7.2937e-02, -2.2327e-01, -7.2670e-03, -1.8115e-01, -3.5156e-01,\n",
       "                      -1.8005e-01, -5.2686e-01, -9.1553e-02, -7.3486e-01, -1.4026e-01,\n",
       "                      -6.7920e-01, -3.0861e-03,  1.6870e-01, -1.0303e-01,  1.1176e-01,\n",
       "                       7.7881e-02, -5.0928e-01, -1.3672e-01, -2.4316e-01, -5.9631e-02,\n",
       "                      -1.8799e-01, -5.9424e-01, -3.8357e-03,  1.3464e-01, -7.5439e-02,\n",
       "                      -4.5801e-01, -2.0020e-01, -2.0508e-01, -6.2598e-01, -2.1143e-01,\n",
       "                      -1.4026e-01, -1.0146e+00, -8.0078e-01, -1.8585e-02, -1.1566e-01,\n",
       "                      -3.4576e-02, -2.3218e-01, -7.7051e-01, -8.7793e-01, -1.1250e+00,\n",
       "                      -1.1755e-01, -1.7517e-01, -5.2344e-01, -1.8945e-01, -5.1758e-01,\n",
       "                      -3.6591e-02, -6.7236e-01, -9.7290e-02, -3.3008e-01, -2.4338e-02,\n",
       "                      -2.3514e-02, -1.9849e-01, -3.0493e-01, -5.4883e-01, -1.8335e-01,\n",
       "                      -8.8379e-02, -1.3171e-01,  1.4783e-01, -2.1753e-01,  2.5244e-01,\n",
       "                       7.9651e-03, -5.9235e-02,  2.3755e-01, -2.9883e-01, -8.8013e-02,\n",
       "                      -3.7231e-01, -4.2191e-03, -1.8774e-01, -4.5471e-02, -1.1765e-02,\n",
       "                      -3.5010e-01,  1.1652e-01,  3.3234e-02, -6.1670e-01, -1.9019e-01,\n",
       "                      -5.9662e-02, -4.1089e-01, -2.0227e-01, -5.8624e-02, -1.4612e-01,\n",
       "                      -1.8835e-01, -1.6699e-01,  2.7054e-02, -1.5625e-01, -1.4697e-01,\n",
       "                       2.4338e-02,  4.0985e-02, -4.5605e-01, -2.5269e-01, -3.2623e-02,\n",
       "                      -3.6011e-01, -9.0576e-02, -1.0156e-01, -2.2449e-01, -5.4207e-03,\n",
       "                      -3.1543e-01, -6.9141e-01, -6.2683e-02, -5.7031e-01, -2.7319e-01,\n",
       "                      -8.7433e-03, -2.7173e-01, -1.4563e-01, -1.9250e-01,  3.8788e-02,\n",
       "                      -1.0010e-01,  1.0498e-01, -3.3813e-01, -2.4646e-01,  2.9831e-02,\n",
       "                      -5.9717e-01, -1.6467e-01,  7.3303e-02, -3.6719e-01, -6.8408e-01,\n",
       "                       1.4197e-01, -1.8787e-01,  1.0818e-02, -3.4570e-01, -1.1414e-02,\n",
       "                      -8.4082e-01, -1.5344e-01, -6.6992e-01, -2.5757e-01,  3.7994e-02,\n",
       "                      -3.2425e-03, -7.7515e-02, -2.9053e-01, -8.9539e-02,  6.5247e-02,\n",
       "                      -8.8959e-03, -5.3223e-02, -2.3633e-01, -2.6611e-01, -1.4453e-01,\n",
       "                      -6.7627e-02,  4.2755e-02, -6.0791e-01,  2.9102e-01, -1.6675e-01,\n",
       "                      -3.9307e-02, -2.3853e-01, -8.1787e-02, -7.3730e-02, -2.3340e-01,\n",
       "                      -7.2070e-01, -2.8394e-01,  1.2537e-01, -2.2961e-01, -2.1667e-01,\n",
       "                      -7.7393e-02, -3.8916e-01,  5.5145e-02,  7.3738e-03, -1.6101e-01,\n",
       "                      -3.9087e-01, -5.4230e-02,  2.4292e-01,  1.3843e-01,  2.2171e-02,\n",
       "                      -4.7882e-02,  8.3313e-02,  4.8340e-02, -5.7520e-01, -2.2232e-02,\n",
       "                      -1.5039e-01, -5.8398e-01, -3.5767e-01,  3.5400e-02, -7.6416e-02,\n",
       "                      -5.6738e-01, -4.9170e-01, -2.1301e-01,  2.9541e-02,  2.1533e-01,\n",
       "                      -6.2500e-02, -3.5919e-02,  2.3834e-02,  1.3782e-01,  2.8900e-02,\n",
       "                       7.7286e-03, -2.0886e-01, -6.1096e-02, -1.1115e-01, -1.3220e-01,\n",
       "                      -3.1860e-02, -2.1143e-01, -1.0315e-01, -4.2017e-01, -2.1338e-01,\n",
       "                      -7.9285e-02, -1.8921e-01, -4.3872e-01, -8.6975e-02, -8.3374e-02,\n",
       "                      -1.7834e-01, -4.7333e-02,  3.4302e-02, -4.5410e-01,  6.5193e-03,\n",
       "                      -2.9327e-02, -3.1860e-01, -4.1626e-01, -1.9116e-01, -1.7749e-01,\n",
       "                      -2.2327e-01, -3.9233e-01, -1.3574e-01, -1.0596e-01,  1.5674e-01,\n",
       "                      -7.5732e-01, -2.9572e-02,  1.7166e-02, -8.1482e-02,  2.4734e-02,\n",
       "                       8.2588e-04, -7.6953e-01, -1.2268e-01, -2.8549e-02, -4.5605e-01,\n",
       "                      -3.6987e-01, -1.9324e-01, -2.6199e-02, -3.0807e-02, -3.8013e-01,\n",
       "                      -7.0801e-02,  6.9946e-02,  3.4973e-02, -9.1858e-02, -3.1006e-02,\n",
       "                      -1.0724e-01,  7.7393e-02, -4.9896e-02, -4.1553e-01, -4.2725e-01,\n",
       "                      -7.9041e-02, -1.0040e-01, -3.4375e-01, -1.2622e-01, -2.1411e-01,\n",
       "                      -2.7979e-01, -1.9263e-01, -3.4766e-01, -5.7715e-01, -3.6816e-01,\n",
       "                      -1.6284e-01, -1.0332e+00, -8.0273e-01,  4.6997e-02,  1.1035e-01,\n",
       "                      -1.7249e-01, -6.9434e-01, -2.4829e-01, -2.0654e-01, -1.0455e-01,\n",
       "                      -7.9468e-02,  1.1334e-01,  1.9897e-01, -4.1107e-02, -1.6077e-01,\n",
       "                      -1.4397e-02, -8.1604e-02, -1.6638e-01, -6.2402e-01, -2.5650e-02,\n",
       "                      -1.3684e-01,  1.7004e-01, -3.2300e-01, -1.6431e-01, -1.5076e-01,\n",
       "                      -1.5125e-01, -1.1707e-01, -1.1273e-01, -4.9316e-02, -9.7717e-02,\n",
       "                      -3.7866e-01, -2.7637e-01, -1.4893e-01, -1.6138e-01, -2.1255e-02,\n",
       "                      -3.1934e-01, -1.2006e-01, -3.3447e-02, -3.3765e-01,  4.8370e-02,\n",
       "                      -1.5576e-01,  7.0992e-03, -1.0474e-01,  3.7903e-02, -2.0862e-01,\n",
       "                      -1.4282e-01, -9.0723e-01, -4.4287e-01, -3.3691e-01, -4.3359e-01,\n",
       "                      -6.2305e-01, -6.7566e-02, -2.6474e-02, -2.0728e-01, -3.1616e-02,\n",
       "                      -8.5205e-02, -6.9641e-02, -8.2275e-02,  2.3804e-01, -5.2765e-02,\n",
       "                      -1.0504e-01,  6.1432e-02,  5.8250e-03, -1.4417e-01,  3.5950e-02,\n",
       "                      -2.1265e-01,  5.8517e-03, -5.2295e-01, -4.7534e-01, -7.6172e-02,\n",
       "                      -1.5149e-01, -5.9180e-01, -4.2407e-01, -5.0488e-01, -1.5503e-01,\n",
       "                      -6.3428e-01, -1.9791e-02,  2.4017e-02, -2.0370e-03, -2.9517e-01,\n",
       "                      -1.6858e-01,  1.8873e-03,  1.6162e-01, -1.9604e-01,  1.7227e-02,\n",
       "                       2.1545e-01,  1.6394e-01, -2.5952e-01, -1.6785e-01, -9.3613e-03,\n",
       "                       1.7004e-01,  5.0964e-02, -1.9470e-01, -2.4011e-01, -1.9312e-01,\n",
       "                      -1.9519e-01, -5.7465e-02, -2.6733e-01, -2.0288e-01, -4.2749e-01,\n",
       "                      -5.3253e-02, -3.4790e-02,  2.2797e-02, -1.4038e-01, -1.4294e-01,\n",
       "                       1.9852e-02, -6.2500e-02,  5.0049e-02, -1.6983e-02, -1.3733e-01,\n",
       "                      -7.4036e-02,  4.1260e-02,  2.7634e-02, -7.7930e-01, -1.8359e-01,\n",
       "                      -1.8848e-01, -7.9639e-01, -8.4473e-02, -7.5806e-02, -2.2144e-01,\n",
       "                      -3.2196e-02, -1.2062e-02, -1.7285e-01, -3.3020e-02, -5.7324e-01,\n",
       "                      -1.2854e-01, -3.0182e-02, -1.8799e-02, -5.2605e-03, -1.8494e-01,\n",
       "                      -4.7925e-01, -3.7378e-01, -4.8004e-02, -1.4062e-01,  4.2267e-02,\n",
       "                      -4.9341e-01,  1.0132e-01,  6.0699e-02,  1.5295e-01, -6.4404e-01,\n",
       "                       2.1802e-01, -8.4167e-02, -3.7964e-01, -7.8613e-02, -1.0138e-01,\n",
       "                      -6.9434e-01, -2.8174e-01, -3.6890e-01, -3.3154e-01, -1.3538e-01,\n",
       "                      -1.2134e-01, -1.1218e-01, -2.3572e-01, -3.8135e-01, -1.5942e-01,\n",
       "                      -1.3550e-01, -1.7712e-01, -1.2354e-01, -3.1006e-01, -9.3018e-02,\n",
       "                      -2.9590e-01, -2.4817e-01, -1.3123e-01, -1.9275e-01,  4.9591e-02,\n",
       "                      -2.7710e-01,  1.4221e-01, -5.3125e-01, -5.6152e-01, -4.6338e-01,\n",
       "                      -3.0786e-01,  1.9836e-03, -2.5610e-01, -1.2891e+00, -6.8207e-03,\n",
       "                      -2.2656e-01, -2.3438e-02, -3.7811e-02,  7.1678e-03, -2.7905e-01,\n",
       "                      -5.1367e-01, -2.0776e-01, -7.5500e-02, -4.1382e-01,  2.2900e-01,\n",
       "                      -4.4873e-01, -7.3792e-02, -5.8691e-01, -6.6162e-02, -2.2815e-01,\n",
       "                      -4.5605e-01, -2.3590e-02,  9.3689e-02, -8.3557e-02, -3.2666e-01,\n",
       "                      -2.2302e-01, -1.9028e-02, -2.2266e-01, -1.4600e-01,  1.1688e-02,\n",
       "                      -2.6172e-01, -1.0242e-01, -7.0114e-03, -5.5713e-01, -4.5703e-01,\n",
       "                      -4.1553e-01, -8.5327e-02, -2.6184e-02, -3.5596e-01, -1.3123e-01,\n",
       "                      -8.7036e-02,  1.4099e-01, -2.6416e-01, -1.2720e-01, -6.2256e-02,\n",
       "                      -2.5635e-01, -2.3035e-01, -5.8075e-02, -4.6539e-02,  3.0930e-02,\n",
       "                      -1.9910e-01, -1.6345e-01,  5.3986e-02, -8.7524e-02,  4.4464e-02,\n",
       "                      -8.0127e-01, -7.5806e-02, -3.1128e-01, -1.0217e-01,  5.3711e-02,\n",
       "                      -2.3209e-02, -7.7393e-02, -3.7451e-01, -7.5928e-01, -1.9495e-01,\n",
       "                      -3.2074e-02, -3.8892e-01,  4.8141e-03,  1.9302e-02, -6.0547e-02,\n",
       "                      -1.8591e-01, -9.0332e-02, -2.4475e-02, -1.4282e-01, -8.9050e-02,\n",
       "                      -1.1676e-01, -2.8589e-01, -2.3950e-01, -2.9810e-01, -1.5076e-01,\n",
       "                      -1.1755e-01, -2.6880e-01,  1.3501e-01,  1.2598e-01, -8.4961e-01,\n",
       "                      -8.0994e-02, -2.5654e-03, -2.6367e-01,  3.0563e-02, -2.6147e-01,\n",
       "                      -1.9873e-01, -7.4512e-01, -1.9312e-01, -5.4346e-01, -3.8599e-01,\n",
       "                       3.5400e-02, -1.2164e-01, -2.9541e-01, -1.5784e-01, -3.3325e-01,\n",
       "                      -4.9805e-02, -1.1456e-01, -4.0698e-01, -7.9834e-02, -6.0577e-02,\n",
       "                      -6.0791e-02, -1.8164e-01, -1.0089e-01, -2.5360e-02])),\n",
       "             ('transformer.h.9.mlp.dense_4h_to_h.weight',\n",
       "              tensor([[-0.0114,  0.0044,  0.0211,  ..., -0.0117,  0.0006,  0.0353],\n",
       "                      [-0.0156, -0.0004, -0.0245,  ...,  0.0236,  0.0007, -0.0339],\n",
       "                      [ 0.0062, -0.0130, -0.0127,  ..., -0.0071, -0.0141, -0.0097],\n",
       "                      ...,\n",
       "                      [-0.0303, -0.0236,  0.0055,  ...,  0.0271,  0.0083, -0.0056],\n",
       "                      [-0.0088,  0.0043, -0.0046,  ..., -0.0243,  0.0142, -0.0081],\n",
       "                      [ 0.0082, -0.0022,  0.0040,  ..., -0.0016,  0.0078, -0.0004]])),\n",
       "             ('transformer.h.9.mlp.dense_4h_to_h.bias',\n",
       "              tensor([ 0.0508, -0.0029, -0.0312,  ..., -0.0634,  0.1105,  0.0009])),\n",
       "             ('transformer.h.10.input_layernorm.weight',\n",
       "              tensor([1.2295, 2.0527, 1.0312,  ..., 2.6113, 2.0137, 0.7422])),\n",
       "             ('transformer.h.10.input_layernorm.bias',\n",
       "              tensor([ 0.0481, -0.0119,  0.0427,  ...,  0.0685,  0.0157,  0.0135])),\n",
       "             ('transformer.h.10.self_attention.query_key_value.weight',\n",
       "              tensor([[-2.0309e-02, -7.9334e-05,  3.2959e-02,  ...,  3.2104e-02,\n",
       "                        5.0240e-03,  3.5248e-02],\n",
       "                      [-1.6220e-02,  4.1870e-02,  1.7624e-03,  ..., -1.2245e-02,\n",
       "                        2.2339e-02,  3.1708e-02],\n",
       "                      [ 5.9700e-03, -3.4302e-02,  3.4149e-02,  ..., -2.3529e-02,\n",
       "                        2.2293e-02,  3.6201e-03],\n",
       "                      ...,\n",
       "                      [ 8.3113e-04,  9.0942e-03,  8.6880e-04,  ..., -1.3992e-02,\n",
       "                       -7.0534e-03, -8.7051e-03],\n",
       "                      [-4.3793e-03,  2.8133e-03, -2.2491e-02,  ..., -2.1027e-02,\n",
       "                       -1.5572e-02,  7.4148e-04],\n",
       "                      [ 2.6260e-02, -2.9099e-02,  2.8442e-02,  ...,  4.8309e-02,\n",
       "                       -1.2314e-02, -2.4429e-02]])),\n",
       "             ('transformer.h.10.self_attention.query_key_value.bias',\n",
       "              tensor([-3.2056e-01,  4.0497e-02, -4.9438e-02,  4.6509e-02,  2.6978e-01,\n",
       "                       8.8867e-02,  1.4563e-01,  1.5656e-02,  8.1848e-02, -7.6416e-01,\n",
       "                       1.9971e-01,  5.5359e-02, -2.0105e-01,  1.5161e-01,  1.6162e-01,\n",
       "                      -1.5002e-01,  3.0103e-01, -1.0114e-01, -2.6270e-01,  1.8884e-01,\n",
       "                       4.6478e-02,  1.6891e-02, -1.1182e-01,  1.7883e-01, -2.0056e-01,\n",
       "                       2.8229e-02,  4.8637e-03,  4.3188e-01, -1.4248e+00, -2.3083e-01,\n",
       "                       1.3293e-01, -1.2048e-01,  8.4152e-03, -3.9764e-02,  2.8955e-01,\n",
       "                       2.8549e-02,  2.0740e-01, -1.3245e-01, -8.5938e-02,  4.0680e-02,\n",
       "                       3.3984e-01, -2.1545e-02, -3.1128e-02,  1.2366e-01, -6.1676e-02,\n",
       "                      -1.4026e-01, -5.6305e-02, -2.3438e-01,  9.8419e-03, -1.6895e-01,\n",
       "                       8.0518e-01,  1.4124e-01, -1.4368e-01,  6.4148e-02,  4.6143e-01,\n",
       "                      -5.8075e-02, -5.4962e-02, -6.4941e-02,  1.7712e-01, -1.1360e-02,\n",
       "                      -3.6957e-02,  1.2646e-01, -3.2104e-01, -1.2657e-02, -1.1584e-01,\n",
       "                       1.3466e-02,  9.8801e-04, -7.5188e-03, -5.0964e-02,  4.1595e-02,\n",
       "                      -1.3977e-02, -1.6495e-02,  3.9792e-04, -3.2007e-01,  7.2388e-02,\n",
       "                      -1.2497e-02, -3.8147e-02, -1.1749e-02, -6.0272e-02, -9.1553e-03,\n",
       "                       8.1116e-02,  4.7211e-02, -4.0436e-02, -1.2199e-02, -1.0852e-01,\n",
       "                      -1.9791e-02, -5.1147e-02, -3.9482e-03, -1.7609e-02, -9.1553e-02,\n",
       "                       3.8116e-02,  1.7554e-01, -7.7148e-01, -2.4216e-02,  7.1144e-03,\n",
       "                       3.5950e-02, -2.5604e-02,  1.2589e-02,  9.1736e-02, -2.2293e-02,\n",
       "                       1.1110e-03,  6.5125e-02, -4.7485e-02, -5.6419e-03,  9.2957e-02,\n",
       "                      -2.0798e-02,  4.5227e-02,  8.2581e-02,  2.0126e-02, -1.0486e-01,\n",
       "                      -1.2444e-02,  2.4048e-02, -8.5144e-02, -8.2336e-02,  1.8091e-01,\n",
       "                      -4.0619e-02, -7.7248e-03, -4.2877e-02,  4.7211e-02, -7.1960e-02,\n",
       "                      -3.3142e-02,  1.0321e-01,  4.5052e-03, -7.1838e-02,  3.6804e-02,\n",
       "                       7.0129e-02, -1.8774e-01, -3.9703e-02,  1.3496e-02, -4.1992e-02,\n",
       "                      -4.9536e-01, -1.2360e-01,  4.2529e-01, -2.1399e-01, -4.8047e+00,\n",
       "                       1.5833e-01, -1.4758e-01,  1.6064e-01,  2.4780e-01,  1.0321e-01,\n",
       "                      -1.3855e-01, -6.3916e-01,  2.5586e-01,  2.8979e-01,  1.4026e-01,\n",
       "                       7.4707e-02, -1.5564e-01, -1.1273e-01, -3.2812e-01, -1.2476e-01,\n",
       "                      -4.6338e-01, -3.3594e-01,  1.0303e-01,  1.8286e-01, -6.0760e-02,\n",
       "                       1.6309e-01, -6.4270e-02, -4.1455e-01,  1.5186e-01, -1.9531e-01,\n",
       "                       2.4780e-01, -6.8817e-03, -4.9019e-03,  2.2937e-01, -2.3303e-01,\n",
       "                      -1.6284e-01,  9.0271e-02, -1.7090e-01, -4.8309e-02,  5.6299e-01,\n",
       "                       1.6772e-01,  8.0338e-03,  1.2079e-01, -2.6367e-01,  6.5820e-01,\n",
       "                      -3.0054e-01, -5.7080e-01,  1.0632e-01, -2.5711e-03,  1.7646e+00,\n",
       "                      -5.0751e-02,  2.7979e-01, -1.0181e-01, -5.9912e-01, -6.2134e-02,\n",
       "                       9.4116e-02,  4.4873e-01, -6.0303e-02, -3.3594e-01, -1.3110e-01,\n",
       "                      -1.5881e-01, -2.1863e-01, -1.1108e-02,  3.2471e-02,  1.1883e-03,\n",
       "                       1.8570e-02,  4.1229e-02,  2.6276e-02, -2.1973e-02,  3.8910e-02,\n",
       "                      -1.9388e-03,  2.2598e-02,  3.0727e-03,  9.5596e-03, -2.7908e-02,\n",
       "                      -2.6367e-02,  1.7761e-02, -2.7084e-03,  3.0945e-02,  3.5126e-02,\n",
       "                      -1.7746e-02, -1.2741e-02,  1.9318e-02, -6.9656e-03,  1.6220e-02,\n",
       "                       1.8524e-02, -3.2463e-03, -2.8915e-03,  2.4567e-02, -1.5671e-02,\n",
       "                       3.8177e-02,  3.1052e-02,  1.4175e-02, -3.0975e-02,  4.4823e-03,\n",
       "                      -1.6006e-02, -1.8555e-02,  1.5625e-02, -1.9638e-02,  1.1192e-02,\n",
       "                       1.2833e-02,  6.0081e-04, -3.3142e-02, -1.6205e-02, -8.0414e-03,\n",
       "                      -1.1110e-03, -3.7098e-03, -8.5220e-03,  7.5340e-03, -1.2836e-03,\n",
       "                      -2.3174e-03,  1.9592e-02,  9.4604e-03, -1.6022e-02,  2.9587e-02,\n",
       "                      -7.9803e-03,  3.2379e-02, -1.6342e-02, -2.4063e-02, -2.1072e-02,\n",
       "                       3.0930e-02,  2.4078e-02,  1.4702e-02, -1.5961e-02,  4.5242e-03,\n",
       "                       1.4824e-02, -1.3652e+00,  2.5215e+00,  7.4414e-01,  2.2119e-01,\n",
       "                      -7.3730e-02,  1.1094e+00,  3.9917e-01, -5.4810e-02,  8.1201e-01,\n",
       "                      -3.5229e-01,  5.8643e-01, -1.2324e+00,  1.0020e+00,  1.3135e+00,\n",
       "                      -4.3335e-01, -3.5278e-01,  7.9834e-01,  3.1299e-01, -2.2095e-01,\n",
       "                       1.2920e+00,  9.5020e-01,  1.2524e-01, -6.7688e-02,  1.3806e-01,\n",
       "                       4.6143e-01,  6.3623e-01,  2.3132e-01, -8.2861e-01,  8.6060e-02,\n",
       "                       1.8079e-01, -7.1045e-01,  2.8735e-01, -1.1096e-01, -1.9609e+00,\n",
       "                      -8.5889e-01,  6.3672e-01, -2.1301e-01,  6.4941e-01,  1.1123e+00,\n",
       "                      -6.1768e-01, -1.8975e+00, -8.9551e-01,  5.5176e-01,  2.7026e-01,\n",
       "                       1.1699e+00,  5.9180e-01,  3.2422e-01,  4.7827e-01,  5.7251e-02,\n",
       "                      -1.2830e-01,  1.8225e-01, -5.9717e-01, -3.8647e-01,  1.1743e-01,\n",
       "                       9.3896e-01, -3.0981e-01,  7.1228e-02, -2.5439e-01,  8.8770e-01,\n",
       "                      -1.1078e-01,  3.0493e-01, -5.1562e-01,  5.0635e-01,  1.4575e-01,\n",
       "                       1.2405e-02, -3.4454e-02, -2.4597e-02, -1.4435e-02,  1.1093e-02,\n",
       "                       2.2678e-03,  2.0065e-03,  2.1683e-02, -8.3313e-03, -2.3308e-03,\n",
       "                       1.4946e-02,  4.7836e-03, -7.4921e-03, -2.6489e-02,  5.4893e-03,\n",
       "                       1.7424e-03,  8.7509e-03,  1.6022e-02,  5.9509e-03,  9.0942e-03,\n",
       "                       1.1963e-02,  2.8275e-02,  1.2177e-02,  5.8708e-03,  1.1806e-03,\n",
       "                       6.7558e-03, -1.4023e-02,  2.1957e-02, -2.6276e-02,  2.2034e-02,\n",
       "                      -3.2471e-02, -4.9553e-03, -5.4245e-03, -2.5368e-03, -3.8452e-02,\n",
       "                      -9.0408e-03, -1.4397e-02, -2.9129e-02, -8.8882e-03, -1.8112e-02,\n",
       "                      -2.1301e-02,  1.3321e-02, -1.1780e-02,  3.9864e-03, -1.5854e-02,\n",
       "                       2.0325e-02,  4.6310e-03, -1.2749e-02,  4.1046e-03,  1.5976e-02,\n",
       "                      -1.3710e-02, -5.7030e-04,  7.4673e-04,  6.9580e-03,  2.4445e-02,\n",
       "                      -6.4373e-05,  9.4910e-03,  6.4430e-03, -7.4280e-02, -1.1208e-02,\n",
       "                      -1.1711e-02,  1.2115e-02, -1.1185e-02,  1.3008e-02,  1.4038e-03,\n",
       "                       1.0574e-02, -5.3406e-03,  3.0930e-02,  1.4977e-02, -1.8661e-02,\n",
       "                       2.9968e-02,  1.5518e-02,  7.9041e-03, -1.5961e-02,  2.2247e-02,\n",
       "                      -1.3184e-02,  6.2418e-04,  4.8103e-03, -5.2910e-03, -3.6373e-03,\n",
       "                      -1.3107e-02, -9.9258e-03, -1.3550e-02,  6.8130e-03, -1.4465e-02,\n",
       "                       2.5120e-03, -9.7656e-03,  1.4214e-02, -2.2324e-02,  3.0899e-03,\n",
       "                      -2.5387e-03,  1.6327e-02, -1.2497e-02, -5.2856e-02,  2.0187e-02,\n",
       "                      -2.0924e-03,  4.0855e-03, -1.7853e-02,  1.4374e-02, -1.8982e-02,\n",
       "                      -3.3455e-03,  1.0933e-02,  7.6965e-02,  2.2221e-03, -1.3123e-02,\n",
       "                      -3.7861e-03, -1.6815e-02,  1.6922e-02,  1.1322e-02,  2.1942e-02,\n",
       "                       9.3002e-03, -1.1997e-03,  2.4529e-03, -1.6068e-02, -1.7914e-02,\n",
       "                       9.0485e-03, -1.2672e-02,  2.6718e-02,  3.3512e-03,  5.2147e-03,\n",
       "                      -2.4643e-03,  1.5669e-03,  5.3368e-03,  5.7144e-03, -1.2238e-02,\n",
       "                      -1.4381e-02, -1.9241e-02, -7.4463e-03, -3.5938e-01,  5.1758e-02,\n",
       "                      -4.6509e-01, -1.9180e-02, -1.1322e-01,  3.5669e-01, -1.1603e-01,\n",
       "                      -1.2634e-01, -1.7480e-01,  2.9321e-01, -7.9053e-01,  1.7261e-01,\n",
       "                       6.2891e-01,  6.7041e-01,  1.3447e+00, -2.5024e-01, -1.0278e-01,\n",
       "                      -2.3682e-02, -5.2917e-02, -5.1758e-01, -7.2510e-01, -5.6543e-01,\n",
       "                       2.5342e-01, -1.3989e-01,  2.0361e-01, -3.1958e-01,  7.1167e-02,\n",
       "                       6.4062e-01,  1.1597e-01, -1.0223e-01, -4.6478e-02,  2.2656e+00,\n",
       "                      -4.8096e-02, -1.0382e-01,  3.8672e-01, -3.0737e-01, -3.1348e-01,\n",
       "                       1.4001e-01, -3.3618e-01, -5.5664e-02, -2.6880e-01,  2.6294e-01,\n",
       "                      -1.4775e+00,  2.8594e+00, -4.3610e-02, -1.0266e-01,  2.5903e-01,\n",
       "                      -3.8428e-01,  2.3901e-01,  4.0723e-01, -3.1567e-01,  1.6235e-01,\n",
       "                       1.0785e-01,  1.7712e-01, -5.5542e-02, -3.2471e-01,  2.6562e-01,\n",
       "                      -1.3135e-01, -3.7622e-01, -5.4443e-01,  3.4595e-01,  2.3633e+00,\n",
       "                      -5.0171e-02,  1.2250e-01, -1.7212e-02, -1.2024e-02,  9.4147e-03,\n",
       "                       5.3253e-03,  3.6640e-03,  2.1606e-02,  2.6138e-02,  2.2171e-02,\n",
       "                       3.9642e-02, -1.7529e-03,  9.2850e-03,  1.2390e-02,  3.6694e-01,\n",
       "                      -2.7637e-03, -1.8875e-02,  1.9913e-03,  1.4534e-02,  9.7351e-03,\n",
       "                       9.9373e-04, -2.3651e-02,  2.3590e-02,  1.7958e-03,  2.2720e-02,\n",
       "                       5.9319e-03, -3.5076e-03,  6.0211e-02, -5.1575e-03,  2.5131e-02,\n",
       "                      -7.2174e-03,  1.6769e-02, -2.3746e-03, -1.4839e-02,  1.1490e-02,\n",
       "                      -2.5291e-03, -1.0902e-02,  2.3956e-02,  2.2827e-02, -2.0264e-02,\n",
       "                       3.4546e-02, -1.1093e-02, -7.5684e-03,  5.5084e-03,  8.5526e-03,\n",
       "                       9.7046e-03, -7.5836e-03, -1.0338e-02, -1.7792e-02, -2.0905e-02,\n",
       "                      -1.7166e-03, -3.3417e-02, -2.4551e-02,  6.8512e-03, -6.3248e-03,\n",
       "                       1.2140e-03, -5.1422e-02, -8.9340e-03, -1.3199e-02,  1.5808e-02,\n",
       "                      -1.0193e-02,  7.3471e-03,  2.5742e-02,  2.8809e-02,  1.5007e-02,\n",
       "                       1.2383e-02,  3.1235e-02,  2.3584e-01, -1.4954e-01, -7.5012e-02,\n",
       "                      -2.1082e-01, -1.8604e-01, -1.4111e-01,  4.0375e-02, -2.3895e-02,\n",
       "                      -3.8916e-01, -1.7798e-01,  3.5962e-01, -1.9519e-01, -3.7537e-02,\n",
       "                      -2.6440e-01,  6.2195e-02, -2.1582e-01,  1.3367e-02, -2.5806e-01,\n",
       "                       8.0383e-02,  3.0933e-01, -5.2071e-03, -1.1011e-01,  1.1859e-01,\n",
       "                      -5.2765e-02,  2.0288e-01,  3.0225e-01, -1.0215e+00, -3.6865e-02,\n",
       "                      -3.8910e-02, -5.5206e-02, -1.9434e+00, -4.3335e-02, -4.1290e-02,\n",
       "                      -4.5654e-02, -7.1526e-03, -2.1802e-01,  3.6377e-02, -1.6650e+00,\n",
       "                      -8.4900e-02, -1.0704e-02,  7.8308e-02,  1.3220e-01, -4.3091e-01,\n",
       "                      -4.1473e-02, -6.2256e-02,  4.9133e-02,  5.3070e-02,  1.8079e-01,\n",
       "                       1.0565e-01,  5.8289e-02, -1.1053e-01,  3.7085e-01,  1.1542e-01,\n",
       "                      -9.8083e-02,  2.5977e-01,  4.5776e-01, -9.0820e-02, -1.1823e-01,\n",
       "                       1.1664e-01, -1.4673e-01,  1.1528e-02,  2.4658e-01,  7.3242e-01,\n",
       "                      -1.4185e-01, -3.9185e-01,  1.9641e-01, -3.7061e-01, -2.5220e-01,\n",
       "                      -3.9990e-01, -9.6582e-01, -7.6172e-01, -2.3425e-01, -7.5781e-01,\n",
       "                      -1.5979e-01, -2.0068e-01, -9.6387e-01,  2.7759e-01, -2.2659e-02,\n",
       "                       3.6836e+00, -8.2666e-01, -1.5088e-01,  2.9517e-01, -9.4775e-01,\n",
       "                       2.0251e-01, -5.6190e-03, -3.3740e-01,  3.7305e-01,  6.2103e-02,\n",
       "                      -2.0764e-01, -7.6843e-02, -1.6660e+00, -3.6475e-01,  1.5015e-01,\n",
       "                       2.7319e-01, -5.7556e-02, -2.3613e+00,  1.6248e-01,  1.6296e-01,\n",
       "                       1.9189e-01,  1.9861e-01,  3.2837e-02, -4.6729e-01, -1.7275e+00,\n",
       "                       1.3135e+00,  5.5878e-02, -1.3711e+00, -4.4922e-01, -1.3086e-01,\n",
       "                       4.5630e-01, -6.8848e-01, -3.8574e-02, -1.6777e+00,  1.0681e-01,\n",
       "                      -6.9580e-02, -4.1992e-01, -2.2473e-01, -1.4905e-01, -4.5068e-01,\n",
       "                       2.3145e+00, -1.2805e-01,  2.9004e-01,  4.6045e-01,  1.7627e+00,\n",
       "                      -4.4727e-01, -4.9780e-01,  2.3169e-01,  6.7627e-02, -1.3445e+01,\n",
       "                      -1.8781e+01,  7.3359e+00, -6.3242e+00,  5.4932e-01, -2.7844e+01,\n",
       "                      -2.5484e+01, -2.5203e+01, -1.3859e+01, -2.5234e+01, -1.3203e+01,\n",
       "                      -1.1594e+01, -2.9188e+01,  1.6438e+01, -1.0576e+00,  6.0781e+01,\n",
       "                      -2.7219e+01, -1.1164e+01,  9.9219e+00, -2.5891e+01,  1.0398e+01,\n",
       "                       1.1125e+01, -2.0234e+01,  2.3188e+01,  1.3492e+01, -4.8789e+00,\n",
       "                      -1.4680e+01, -4.0969e+01, -2.0953e+01,  1.6031e+01,  1.3977e+01,\n",
       "                      -1.0570e+01, -5.6844e+01, -5.5742e+00,  6.3906e+00,  7.7305e+00,\n",
       "                       1.2672e+01,  7.5898e+00, -1.8469e+01, -4.0906e+01,  3.5594e+01,\n",
       "                       6.1992e+00, -3.6969e+01, -9.8828e+00, -1.0414e+01,  2.5234e+01,\n",
       "                      -3.0922e+01,  5.2031e+00, -4.1938e+01, -1.3039e+01, -7.3906e+00,\n",
       "                      -2.4625e+01, -1.4195e+01,  6.6172e+00, -1.5109e+01,  5.8219e+01,\n",
       "                      -1.0820e+01,  9.3750e+00,  2.0797e+01,  3.6562e+01, -6.1562e+00,\n",
       "                      -2.0469e+01,  1.0164e+01,  2.5801e+00])),\n",
       "             ('transformer.h.10.self_attention.dense.weight',\n",
       "              tensor([[-0.0057, -0.0100, -0.0156,  ...,  0.0066,  0.0127, -0.0036],\n",
       "                      [-0.0092,  0.0237, -0.0216,  ..., -0.0148,  0.0111,  0.0168],\n",
       "                      [ 0.0070, -0.0026, -0.0043,  ..., -0.0166, -0.0055,  0.0158],\n",
       "                      ...,\n",
       "                      [-0.0129,  0.0199,  0.0065,  ..., -0.0275,  0.0169, -0.0166],\n",
       "                      [ 0.0023, -0.0373,  0.0257,  ..., -0.0002, -0.0011,  0.0096],\n",
       "                      [ 0.0078,  0.0015,  0.0020,  ..., -0.0124, -0.0068,  0.0082]])),\n",
       "             ('transformer.h.10.self_attention.dense.bias',\n",
       "              tensor([-0.0034,  0.0037,  0.0246,  ..., -0.0182, -0.0297,  0.0161])),\n",
       "             ('transformer.h.10.post_attention_layernorm.weight',\n",
       "              tensor([1.0430, 1.3389, 0.9048,  ..., 1.7305, 1.5576, 0.6235])),\n",
       "             ('transformer.h.10.post_attention_layernorm.bias',\n",
       "              tensor([0.0382, 0.0629, 0.0972,  ..., 0.0106, 0.0658, 0.0474])),\n",
       "             ('transformer.h.10.mlp.dense_h_to_4h.weight',\n",
       "              tensor([[-3.2711e-03,  3.5858e-02,  5.0201e-03,  ..., -1.8478e-02,\n",
       "                        1.0887e-02,  1.7624e-02],\n",
       "                      [ 2.3782e-05, -1.2924e-02,  3.2898e-02,  ...,  1.6909e-03,\n",
       "                        7.9346e-03,  4.7722e-03],\n",
       "                      [-1.9684e-02, -4.8004e-02, -3.8147e-02,  ...,  1.8644e-03,\n",
       "                       -4.7455e-02,  9.9869e-03],\n",
       "                      ...,\n",
       "                      [-8.9884e-04,  9.6741e-03, -3.1799e-02,  ...,  1.2878e-02,\n",
       "                       -5.8899e-03,  2.2827e-02],\n",
       "                      [-7.4911e-04,  5.0125e-03,  5.6763e-03,  ..., -5.8479e-03,\n",
       "                        1.6846e-02, -2.0874e-02],\n",
       "                      [ 2.7790e-03, -3.4637e-02, -2.3880e-02,  ...,  7.4291e-04,\n",
       "                        8.3084e-03,  5.0621e-03]])),\n",
       "             ('transformer.h.10.mlp.dense_h_to_4h.bias',\n",
       "              tensor([-0.4731,  0.2163, -0.5308,  ..., -0.3564, -0.0142, -0.4602])),\n",
       "             ('transformer.h.10.mlp.dense_4h_to_h.weight',\n",
       "              tensor([[ 0.0075,  0.0069, -0.0072,  ..., -0.0013, -0.0043,  0.0110],\n",
       "                      [ 0.0102,  0.0009,  0.0061,  ..., -0.0077, -0.0173, -0.0032],\n",
       "                      [-0.0018, -0.0169, -0.0159,  ..., -0.0045,  0.0008, -0.0261],\n",
       "                      ...,\n",
       "                      [-0.0396, -0.0107,  0.0085,  ...,  0.0044,  0.0240, -0.0035],\n",
       "                      [ 0.0080, -0.0068, -0.0287,  ..., -0.0198, -0.0100,  0.0077],\n",
       "                      [ 0.0045,  0.0105,  0.0078,  ...,  0.0249, -0.0014,  0.0062]])),\n",
       "             ('transformer.h.10.mlp.dense_4h_to_h.bias',\n",
       "              tensor([ 0.0607, -0.0528, -0.0630,  ..., -0.2651,  0.1812,  0.0376])),\n",
       "             ('transformer.h.11.input_layernorm.weight',\n",
       "              tensor([1.1494, 1.6230, 1.0498,  ..., 2.0840, 1.6826, 0.8306])),\n",
       "             ('transformer.h.11.input_layernorm.bias',\n",
       "              tensor([ 0.0123,  0.0612,  0.0665,  ...,  0.0712,  0.0276, -0.0184])),\n",
       "             ('transformer.h.11.self_attention.query_key_value.weight',\n",
       "              tensor([[-0.0325, -0.0062, -0.0057,  ...,  0.0008, -0.0111, -0.0565],\n",
       "                      [ 0.0040, -0.0286,  0.0231,  ...,  0.0077,  0.0190, -0.0252],\n",
       "                      [-0.0102,  0.0276, -0.0399,  ..., -0.0684, -0.0162,  0.0067],\n",
       "                      ...,\n",
       "                      [ 0.0232, -0.0141,  0.0163,  ..., -0.0235,  0.0024, -0.0211],\n",
       "                      [-0.0083, -0.0001,  0.0065,  ..., -0.0030, -0.0077, -0.0179],\n",
       "                      [-0.0031,  0.0039,  0.0030,  ..., -0.0091, -0.0072,  0.0325]])),\n",
       "             ('transformer.h.11.self_attention.query_key_value.bias',\n",
       "              tensor([ 1.4221e-01,  1.0614e-01,  9.1248e-02, -4.0512e-03, -1.4145e-02,\n",
       "                      -1.4969e-02,  5.7587e-02,  1.1993e-01, -8.3252e-02, -1.8494e-02,\n",
       "                       5.0278e-03, -1.0612e-02, -3.3112e-02,  2.7145e-02, -1.4392e-01,\n",
       "                      -7.2556e-03,  1.3184e-01,  4.7333e-02, -1.3275e-02, -1.5845e-01,\n",
       "                      -4.8096e-02,  8.8928e-02,  2.7878e-02, -2.7100e-02, -8.4106e-02,\n",
       "                      -3.6560e-02, -2.7969e-02,  1.9348e-02, -8.9539e-02, -1.1304e-01,\n",
       "                      -1.3135e-01, -9.1248e-02, -7.1350e-02, -2.8214e-02, -4.5471e-02,\n",
       "                      -3.6499e-02, -1.1603e-01, -6.6223e-02,  5.9723e-02, -9.2224e-02,\n",
       "                       4.0466e-02, -3.5858e-02, -5.4016e-02, -5.9387e-02,  9.7107e-02,\n",
       "                      -5.4657e-02,  2.6764e-02,  9.9170e-01,  8.6731e-02,  7.0007e-02,\n",
       "                      -2.6474e-02,  3.5126e-02,  2.3938e-01,  5.9662e-02, -5.9967e-02,\n",
       "                       9.3384e-02, -3.9459e-02, -4.2389e-02, -8.5205e-02, -1.3885e-03,\n",
       "                       8.6426e-02,  7.8125e-02, -2.6062e-02,  1.1145e-01,  1.5381e-01,\n",
       "                      -1.4722e-01,  2.7542e-02,  8.5791e-01, -3.7891e-01, -2.8882e-01,\n",
       "                       4.3140e-01, -1.9373e-01, -5.1367e-01, -1.6785e-01,  1.0156e-01,\n",
       "                      -2.8671e-02,  2.5903e-01,  2.2815e-01,  9.4482e-02, -9.8572e-02,\n",
       "                       1.9355e+00, -6.9275e-02,  6.1279e-01, -1.8054e-01,  2.0349e-01,\n",
       "                       4.8413e-01, -1.9763e-01,  5.3955e-01,  8.3847e-03, -3.1836e-01,\n",
       "                      -1.5283e-01, -1.1969e-01, -1.3098e-01,  1.9458e-01, -3.0249e-01,\n",
       "                      -1.1505e-01,  1.9211e-02, -3.3179e-01, -6.7505e-02,  1.9849e-01,\n",
       "                      -1.3232e-01,  7.9407e-02,  2.7878e-02,  8.0627e-02,  2.1082e-01,\n",
       "                       1.2482e-01, -3.7598e-01, -2.7271e-01, -6.2744e-02, -1.6492e-01,\n",
       "                      -3.2593e-01, -2.3108e-01,  2.1765e-01,  6.6650e-02, -1.6931e-01,\n",
       "                      -2.2571e-01,  2.2446e-02,  3.2749e-03,  2.1309e+00, -8.5449e-02,\n",
       "                       4.2480e-01,  4.1968e-01, -1.7725e-01,  1.3708e-01,  1.7334e-01,\n",
       "                      -1.4771e-01, -5.4102e-01, -2.3572e-01,  1.1955e-02, -2.9236e-02,\n",
       "                      -1.1833e-02,  1.1848e-02,  2.6367e-02,  6.9092e-02, -1.4160e-02,\n",
       "                       4.2076e-03, -4.1321e-02, -3.0121e-02,  7.4577e-04,  1.1818e-02,\n",
       "                       7.7637e-02, -3.6255e-02,  4.0070e-02, -6.9397e-02,  3.9330e-03,\n",
       "                       3.1006e-02,  4.2328e-02,  4.2877e-02,  9.5032e-02, -1.6830e-02,\n",
       "                      -3.4302e-02,  2.9160e-02,  5.0751e-02,  2.1713e-02,  1.2083e-03,\n",
       "                      -2.4155e-02, -1.3878e-02, -1.1459e-02,  4.0344e-02, -1.0941e-02,\n",
       "                       2.0706e-02, -3.2928e-02,  1.6983e-02,  7.0763e-03,  7.1983e-03,\n",
       "                      -2.4353e-02, -1.2405e-02, -2.4658e-02,  1.3855e-02, -6.8130e-03,\n",
       "                       2.7222e-02,  2.9739e-02,  7.7698e-02,  3.5095e-02,  2.7924e-02,\n",
       "                      -3.7048e-02, -5.8777e-02, -6.0196e-03,  3.0106e-02,  4.7028e-02,\n",
       "                       1.3519e-02,  1.7900e-03,  5.7953e-02,  1.3969e-02,  1.8433e-02,\n",
       "                      -3.1021e-02,  3.5797e-02,  3.5950e-02, -2.1133e-02,  5.1514e-02,\n",
       "                      -5.0873e-02,  3.8574e-02,  3.3350e-01,  7.0312e-01,  5.5859e-01,\n",
       "                      -4.5142e-01,  3.3643e-01, -1.6077e-01, -3.2788e-01, -1.6266e-02,\n",
       "                      -3.0640e-01,  3.3423e-01,  3.7476e-01, -1.2817e-01,  9.4788e-02,\n",
       "                      -3.1152e-01, -6.0150e-02,  2.8125e-01,  1.2323e-01,  1.9092e-01,\n",
       "                      -2.7808e-01, -7.8223e-01, -1.2756e-01,  5.5908e-02,  4.8389e-01,\n",
       "                      -2.0081e-01,  6.2752e-03, -5.2002e-01, -1.5637e-01,  1.9971e-01,\n",
       "                       8.9233e-02, -1.0425e-01,  7.6477e-02,  5.3864e-02,  1.7242e-02,\n",
       "                      -4.5410e-02,  1.6882e-01,  6.6797e-01,  2.9761e-01, -1.7578e-01,\n",
       "                      -4.3018e-01,  2.6807e-01, -1.3550e-01,  4.3506e-01, -2.5049e-01,\n",
       "                      -7.2754e-02,  1.1748e+00, -1.6455e-01, -1.7932e-01,  4.4067e-01,\n",
       "                      -1.2830e-01,  3.0615e-01,  1.5759e-01, -4.5581e-01, -2.1497e-01,\n",
       "                       3.3740e-01,  6.3110e-02, -2.0007e-01, -1.9373e-01, -2.6807e-01,\n",
       "                      -6.1249e-02,  1.6494e+00,  5.5566e-01, -2.4512e-01, -1.8037e+00,\n",
       "                       1.6577e-01, -3.9101e-03,  2.2049e-02, -1.0521e-02, -3.0624e-02,\n",
       "                       1.7090e-02, -4.3610e-02, -4.4159e-02,  4.3579e-02, -2.5330e-02,\n",
       "                       2.6627e-02, -7.0724e-03,  1.4906e-03, -2.4223e-03, -3.5217e-02,\n",
       "                       7.8583e-03, -4.1565e-02, -1.7014e-02, -2.0096e-02, -1.1673e-03,\n",
       "                       2.7714e-03,  1.4793e-02,  3.0914e-02,  7.9803e-03,  2.1820e-02,\n",
       "                       1.4267e-02, -1.5312e-02, -1.3704e-03,  1.6144e-02,  3.0251e-03,\n",
       "                      -3.0319e-02,  1.1597e-02, -9.5415e-04, -5.5817e-02,  2.9724e-02,\n",
       "                       2.9160e-02,  6.1417e-03,  9.5978e-03,  6.6032e-03, -2.4887e-02,\n",
       "                      -4.1986e-04,  6.0539e-03, -2.7634e-02,  8.0414e-03, -2.8732e-02,\n",
       "                      -4.1321e-02, -4.4281e-02,  1.4641e-02, -3.3020e-02,  2.0554e-02,\n",
       "                       2.6031e-02,  1.6403e-02, -5.4588e-03, -7.5760e-03, -1.4328e-02,\n",
       "                       3.3783e-02, -2.5421e-02,  6.9046e-03, -1.2856e-02,  1.0284e-02,\n",
       "                       8.4534e-02, -2.1530e-02,  2.3682e-02,  5.1147e-02, -3.0075e-02,\n",
       "                       4.4360e-01,  4.3457e-01,  2.4824e+00,  1.7871e+00,  8.2092e-03,\n",
       "                       4.2065e-01, -6.0156e+00, -8.6523e-01, -9.9805e-01, -5.3760e-01,\n",
       "                       5.8252e-01, -8.6914e-01,  3.0640e-01,  1.4180e+00,  8.4141e+00,\n",
       "                      -7.3535e-01,  5.4346e-01,  3.8516e+00,  5.5859e-01,  6.5625e-01,\n",
       "                       1.2373e+00, -2.2314e-01, -4.4482e-01, -2.0508e+00, -1.5955e-01,\n",
       "                      -7.2217e-01, -1.2627e+00,  9.4873e-01,  3.5791e-01,  7.0264e-01,\n",
       "                      -1.3955e+00,  1.3584e+00,  1.9678e-01,  1.1348e+00, -7.1973e-01,\n",
       "                       9.4629e-01, -9.3262e-01,  3.3295e-02, -9.8486e-01,  5.0879e-01,\n",
       "                      -9.9365e-01,  1.5137e+00, -5.4359e-03,  7.3281e+00,  1.1420e-01,\n",
       "                      -1.1040e-02,  9.6289e-01, -1.2207e-01, -1.4453e+00, -3.1958e-01,\n",
       "                       1.9326e+00, -2.1113e+00,  5.4150e-01,  7.2812e+00,  1.5186e+00,\n",
       "                       1.1816e+00,  4.7290e-01, -5.6982e-01, -7.2070e-01, -2.7100e-01,\n",
       "                      -5.0732e-01, -1.0312e+00,  1.5752e+00, -4.2432e-01, -2.7832e-01,\n",
       "                      -7.1655e-02, -7.3047e-01, -2.1704e-01, -1.0516e-01,  8.3435e-02,\n",
       "                      -5.5275e-03, -7.5378e-02, -2.3169e-01,  4.9072e-01,  9.3945e-01,\n",
       "                       7.3291e-01,  1.9629e+00, -4.8291e-01,  5.5176e-01, -6.2439e-02,\n",
       "                      -2.9077e-01,  1.5613e-01,  8.1253e-03,  9.5703e-02,  2.0898e-01,\n",
       "                      -1.7920e-01,  2.4414e-01, -2.2070e+00,  3.2642e-01,  5.1172e-01,\n",
       "                       5.0928e-01, -2.3914e-01, -2.0178e-01,  4.2847e-01, -4.4891e-02,\n",
       "                      -1.9922e-01,  1.2520e+00,  5.4980e-01, -7.6025e-01, -1.9121e+00,\n",
       "                       3.6469e-02,  8.6609e-02, -3.8550e-01,  2.7856e-01,  3.3862e-01,\n",
       "                      -3.2690e-01, -1.4648e-01,  1.6589e-01, -1.9397e-01,  2.5024e-01,\n",
       "                       5.8203e-01,  1.6602e-01,  3.6060e-01, -4.7192e-01, -4.9316e-02,\n",
       "                       1.6464e-02,  2.4102e+00, -3.4741e-01, -1.6772e-01,  1.9766e+00,\n",
       "                      -3.0298e-01,  1.8396e-01, -1.7588e+00,  2.6904e-01,  6.8787e-02,\n",
       "                       1.6235e-01,  2.2559e+00, -6.4758e-02, -7.4463e-01, -5.2539e-01,\n",
       "                      -2.1074e+00, -1.0645e+00,  3.3105e-01,  3.2886e-01,  1.2861e+00,\n",
       "                      -1.4043e+00, -1.5247e-01,  1.6345e-01,  2.2227e+00,  1.5645e+00,\n",
       "                       5.6445e+00,  7.0496e-02,  1.8867e+00,  2.5391e-01, -2.2766e-01,\n",
       "                      -1.7810e-01, -1.0088e+00,  1.0801e+00, -1.1318e+00, -8.6523e-01,\n",
       "                      -2.4829e-01, -6.4883e+00,  1.3916e-01,  4.7876e-01,  1.8936e+00,\n",
       "                       7.0557e-01, -1.2207e+00,  1.5146e+00,  3.1958e-01,  5.1074e-01,\n",
       "                       3.5664e+00,  1.3545e+00, -1.6846e+00, -5.8750e+00, -2.6562e-01,\n",
       "                       1.1045e+00, -1.3867e+00,  1.7480e+00,  5.0342e-01, -5.8350e-01,\n",
       "                      -2.2969e+00, -2.3340e-01, -1.8262e-01,  1.5198e-01,  1.7910e+00,\n",
       "                       1.1816e+00,  1.0332e+00, -5.5713e-01,  4.2236e-01, -8.2471e-01,\n",
       "                       6.6562e+00, -7.7441e-01, -1.2812e+00,  5.7852e+00, -3.0371e+00,\n",
       "                      -8.8574e-01, -4.8242e+00,  2.7383e+00, -1.6162e-01,  6.7969e-01,\n",
       "                       7.5039e+00, -2.4023e+00, -2.2144e-01,  1.9795e+00,  3.2196e-02,\n",
       "                       4.2572e-03,  1.5259e-01,  2.5317e-01,  2.0586e+00, -9.4604e-02,\n",
       "                      -1.3892e-01,  8.5938e-02, -1.8631e-02, -1.8640e-01,  9.3018e-02,\n",
       "                      -2.7515e-01,  1.8829e-02, -1.0052e-01, -1.3965e-01,  1.6187e-01,\n",
       "                      -1.9397e-01, -8.5693e-02,  5.0507e-03,  1.4355e-01,  3.4399e-01,\n",
       "                       6.5552e-02, -7.9346e-02, -1.1133e-01,  6.2500e-02, -1.3452e-01,\n",
       "                      -1.4795e-01, -1.4633e-02, -4.8103e-03, -2.2827e-01, -9.4287e-01,\n",
       "                      -8.9172e-02, -1.0841e-02,  1.0089e-01, -2.0959e-01, -2.7026e-01,\n",
       "                      -4.4238e-01,  3.2104e-01,  7.8918e-02, -1.0146e+00, -4.1602e-01,\n",
       "                      -1.1523e-01, -3.8727e-02, -4.1406e-01,  6.5491e-02,  1.6357e-01,\n",
       "                       1.1792e-01, -3.1421e-01, -1.7793e+00,  1.1169e-01,  5.9570e-02,\n",
       "                      -8.3594e-01,  1.8274e-01,  1.3770e-01,  7.1350e-02,  1.8213e-01,\n",
       "                       1.9958e-01,  2.0459e-01,  9.0576e-02, -1.8433e-02, -4.1187e-01,\n",
       "                       4.7900e-01, -1.4221e-02,  3.3936e-02,  6.6414e-03,  2.0264e-02,\n",
       "                       1.3571e-03, -3.5370e-02, -6.6895e-02,  5.9875e-02,  1.5808e-02,\n",
       "                       3.9368e-02, -7.4890e-02,  5.9357e-02, -9.2773e-03,  4.2664e-02,\n",
       "                       2.3087e-02,  4.6570e-02, -3.4271e-02,  1.3275e-02, -4.6387e-02,\n",
       "                      -1.8738e-02, -1.6144e-02, -5.5145e-02,  4.2236e-02, -2.8107e-02,\n",
       "                      -2.5772e-02, -4.6783e-02, -5.0110e-02,  2.9144e-02, -3.6987e-02,\n",
       "                       2.4277e-02,  4.6234e-02,  3.6133e-02, -2.8305e-02, -6.1951e-02,\n",
       "                      -5.7404e-02,  5.8380e-02,  1.7883e-02, -2.1347e-02,  1.4671e-02,\n",
       "                      -4.7668e-02,  3.6621e-03,  3.7537e-03, -2.7298e-02, -2.8305e-02,\n",
       "                       4.7729e-02,  2.1057e-02,  6.3293e-02,  3.8574e-02,  1.3489e-02,\n",
       "                      -8.2855e-03,  1.6708e-02,  6.4819e-02, -3.7842e-02,  4.9774e-02,\n",
       "                       3.1433e-02, -7.7343e-04,  8.2336e-02,  5.3680e-02,  4.7699e-02,\n",
       "                      -2.8580e-02,  7.3669e-02,  5.9967e-02, -1.4854e-02, -4.6021e-02,\n",
       "                      -4.2456e-01,  3.7812e+00,  6.3125e+00, -2.5406e+01,  1.7891e+01,\n",
       "                       4.4062e+00,  1.4229e+00,  1.6816e+00,  5.9883e+00,  2.9141e+00,\n",
       "                       2.7578e+00, -1.3916e+00,  1.7480e+00,  5.1611e-01, -6.0508e+00,\n",
       "                      -1.8252e+00,  1.2617e+00, -1.8594e+01, -2.9160e+00, -6.8477e+00,\n",
       "                      -2.6031e+01,  4.7852e+00, -1.9707e+00, -1.1064e+00, -3.6304e-01,\n",
       "                       7.3438e+00,  5.3633e+00, -4.3359e-01, -1.0820e+01, -6.5273e+00,\n",
       "                      -3.5801e+00,  1.0217e-01, -1.1383e+01, -3.8965e+00, -8.8318e-02,\n",
       "                       1.0750e+01,  3.9707e+00,  3.3262e+00, -1.0376e-01,  6.9629e-01,\n",
       "                       4.7969e+00, -8.7891e+00, -8.2656e+00, -1.4512e+00, -3.8477e+00,\n",
       "                      -3.6797e+00, -6.6484e+00, -2.5195e+00, -7.4844e+00,  4.8203e+00,\n",
       "                      -3.5000e+00, -3.2715e-01,  9.4531e+00, -1.5400e+00, -9.4609e+00,\n",
       "                      -1.2602e+01, -3.5879e+00,  1.7141e+01, -8.6250e+00, -4.1758e+00,\n",
       "                      -6.9180e+00,  4.2305e+00, -3.8562e+01, -3.7070e+00, -1.9073e-02,\n",
       "                      -2.2278e-03,  1.2886e-02,  2.4307e-02, -1.2405e-02,  1.2657e-02,\n",
       "                       2.1988e-02,  1.3676e-03, -4.3152e-02,  3.4637e-02,  1.9745e-02,\n",
       "                       3.2349e-02, -7.7009e-04,  1.4143e-03,  7.2670e-03,  2.5192e-02,\n",
       "                       9.9976e-02,  3.8635e-02, -5.4512e-03, -1.6754e-02,  5.5176e-02,\n",
       "                      -4.7150e-03,  1.4633e-02,  1.0614e-03,  7.6050e-02, -2.5146e-02,\n",
       "                       1.3245e-02, -6.6650e-02,  2.8648e-03, -8.6260e-04,  1.3229e-02,\n",
       "                      -2.5368e-03,  1.3113e-03, -1.4282e-02,  1.8265e-02, -1.7939e-03,\n",
       "                      -5.9326e-01,  6.5186e-02, -4.0932e-03, -6.0669e-02, -7.7393e-02,\n",
       "                      -5.3955e-02,  2.7542e-02,  7.0992e-03, -9.1095e-03,  7.5760e-03,\n",
       "                       5.9853e-03,  8.9493e-03, -1.9852e-02, -2.0523e-02, -2.0615e-02,\n",
       "                      -2.1164e-02, -6.9046e-03,  1.1223e-02, -2.0096e-02, -2.5772e-02,\n",
       "                      -9.1019e-03, -2.5681e-02,  2.2400e-02, -2.6703e-02,  1.3710e-02,\n",
       "                       3.2562e-02,  3.5156e-02, -2.1027e-02])),\n",
       "             ('transformer.h.11.self_attention.dense.weight',\n",
       "              tensor([[-0.0024, -0.0097,  0.0125,  ...,  0.0288, -0.0213,  0.0243],\n",
       "                      [ 0.0040,  0.0296,  0.0112,  ..., -0.0052, -0.0080, -0.0068],\n",
       "                      [ 0.0059, -0.0154, -0.0056,  ..., -0.0055,  0.0065,  0.0007],\n",
       "                      ...,\n",
       "                      [ 0.0091, -0.0106, -0.0180,  ..., -0.0142, -0.0053, -0.0130],\n",
       "                      [-0.0127, -0.0060, -0.0040,  ...,  0.0027, -0.0128, -0.0059],\n",
       "                      [-0.0039, -0.0051, -0.0006,  ..., -0.0164, -0.0208, -0.0081]])),\n",
       "             ('transformer.h.11.self_attention.dense.bias',\n",
       "              tensor([-0.0136,  0.0067, -0.0154,  ..., -0.1174, -0.0466,  0.0250])),\n",
       "             ('transformer.h.11.post_attention_layernorm.weight',\n",
       "              tensor([1.0986, 1.2744, 0.9473,  ..., 1.5605, 1.4990, 0.7266])),\n",
       "             ('transformer.h.11.post_attention_layernorm.bias',\n",
       "              tensor([ 0.0737,  0.0441,  0.0961,  ..., -0.0297,  0.1154,  0.0282])),\n",
       "             ('transformer.h.11.mlp.dense_h_to_4h.weight',\n",
       "              tensor([[-0.0156,  0.0112,  0.0031,  ..., -0.0175, -0.0049, -0.0183],\n",
       "                      [ 0.0053,  0.0232, -0.0015,  ...,  0.0071, -0.0018,  0.0435],\n",
       "                      [-0.0231,  0.0168,  0.0105,  ..., -0.0049,  0.0004,  0.0109],\n",
       "                      ...,\n",
       "                      [-0.0137, -0.0184,  0.0123,  ..., -0.0210, -0.0370,  0.0153],\n",
       "                      [ 0.0226, -0.0194,  0.0244,  ...,  0.0067,  0.0092,  0.0235],\n",
       "                      [-0.0053, -0.0081, -0.0318,  ...,  0.0117,  0.0055,  0.0003]])),\n",
       "             ('transformer.h.11.mlp.dense_h_to_4h.bias',\n",
       "              tensor([-0.0253, -0.0354, -0.1503,  ..., -0.0429, -0.2152,  0.0296])),\n",
       "             ('transformer.h.11.mlp.dense_4h_to_h.weight',\n",
       "              tensor([[ 0.0268, -0.0035,  0.0232,  ...,  0.0077,  0.0049,  0.0073],\n",
       "                      [-0.0078, -0.0159,  0.0183,  ..., -0.0199,  0.0051,  0.0079],\n",
       "                      [ 0.0074,  0.0120, -0.0024,  ..., -0.0075,  0.0172,  0.0147],\n",
       "                      ...,\n",
       "                      [ 0.0048, -0.0115,  0.0232,  ..., -0.0246,  0.0077,  0.0231],\n",
       "                      [ 0.0167, -0.0136, -0.0047,  ..., -0.0054, -0.0220,  0.0244],\n",
       "                      [ 0.0038, -0.0051, -0.0078,  ...,  0.0108, -0.0016, -0.0111]])),\n",
       "             ('transformer.h.11.mlp.dense_4h_to_h.bias',\n",
       "              tensor([ 0.0251, -0.0581, -0.1246,  ..., -0.2295,  0.2139,  0.0085])),\n",
       "             ('transformer.h.12.input_layernorm.weight',\n",
       "              tensor([1.2842, 1.9189, 1.1152,  ..., 2.3691, 1.9141, 0.8003])),\n",
       "             ('transformer.h.12.input_layernorm.bias',\n",
       "              tensor([ 0.0115,  0.0129,  0.0443,  ...,  0.0965,  0.0347, -0.0222])),\n",
       "             ('transformer.h.12.self_attention.query_key_value.weight',\n",
       "              tensor([[-0.0146, -0.0214, -0.0015,  ...,  0.0141,  0.0264,  0.0098],\n",
       "                      [-0.0082, -0.0154,  0.0216,  ..., -0.0190,  0.0084, -0.0121],\n",
       "                      [-0.0098,  0.0207,  0.0060,  ..., -0.0071, -0.0004,  0.0143],\n",
       "                      ...,\n",
       "                      [-0.0314,  0.0205,  0.0064,  ...,  0.0146, -0.0040,  0.0011],\n",
       "                      [-0.0206,  0.0040, -0.0061,  ..., -0.0394,  0.0271, -0.0049],\n",
       "                      [-0.0370, -0.0305,  0.0023,  ...,  0.0432, -0.0013,  0.0227]])),\n",
       "             ('transformer.h.12.self_attention.query_key_value.bias',\n",
       "              tensor([-1.5312e-02, -2.7557e-02, -1.2323e-01, -2.8882e-01, -1.5282e-02,\n",
       "                      -4.5239e-01,  2.9443e-01,  3.3350e-01, -1.0242e-01,  1.2666e+00,\n",
       "                       9.6802e-02,  2.6758e-01, -1.1749e-01, -1.1469e-01,  2.4567e-02,\n",
       "                      -7.6904e-03,  9.2041e-02, -1.8408e-01, -3.3112e-02, -7.1228e-02,\n",
       "                       2.8223e-01,  2.1240e-01,  2.4200e-02, -1.0596e-01, -4.0161e-02,\n",
       "                       2.8613e-01,  2.4658e-01,  2.0081e-01, -6.7078e-02, -2.9004e-01,\n",
       "                       7.0984e-02, -4.0558e-02, -7.3438e-01, -4.3701e-02, -7.1960e-02,\n",
       "                       1.7566e-01, -7.5439e-02, -4.7363e-02,  4.1748e-01,  5.6854e-02,\n",
       "                       1.7664e-01,  1.0719e-02,  1.4722e-01,  1.7017e-01,  3.3264e-02,\n",
       "                       1.9543e-01,  1.1841e-01, -1.4587e-01,  8.5815e-02,  5.4741e-03,\n",
       "                       7.3975e-02, -4.8279e-02,  2.3962e-01, -1.4053e-02,  3.6987e-01,\n",
       "                       2.0569e-01, -1.0529e-01, -2.6929e-01,  5.1697e-02,  4.4006e-02,\n",
       "                      -5.9143e-02,  2.5488e-01,  5.8136e-02,  2.2988e+00,  1.5283e-01,\n",
       "                      -1.2610e-01,  1.9763e-01,  1.2109e-01,  1.0577e-01,  6.5613e-02,\n",
       "                      -5.2734e-02, -2.3026e-02, -9.0332e-02, -9.0942e-02,  1.2207e-01,\n",
       "                       1.1755e-01,  4.3182e-02,  1.3208e-01, -1.0974e-01, -4.0649e-02,\n",
       "                       1.8628e-01, -3.5217e-02,  2.4757e-03,  6.0303e-01, -7.8223e-01,\n",
       "                      -6.5857e-02,  1.3379e-01,  5.6671e-02, -6.0669e-02,  2.9129e-02,\n",
       "                      -9.7290e-02,  5.8258e-02, -5.0690e-02, -8.1329e-03, -1.0913e-01,\n",
       "                      -8.5632e-02,  1.0419e-01,  2.0721e-02, -2.0056e-01,  1.2225e-01,\n",
       "                       3.0624e-02, -5.1392e-02, -5.1178e-02,  7.0572e-03, -3.2471e-02,\n",
       "                      -2.5528e-02, -2.7539e-01, -3.4515e-02, -6.4209e-02, -9.0088e-02,\n",
       "                      -7.7576e-02,  8.9417e-02,  1.2140e-01, -1.1969e-01, -1.2666e+00,\n",
       "                       7.9712e-02, -6.3110e-02, -1.4775e+00, -8.5327e-02, -9.3445e-02,\n",
       "                       1.3786e-02,  7.3204e-03,  4.6143e-02, -6.6566e-03, -1.9788e-01,\n",
       "                      -8.6487e-02, -5.8708e-03, -2.1704e-01, -1.9135e-02, -3.0117e-03,\n",
       "                      -1.8433e-02, -1.1787e-02,  3.0258e-02,  5.1361e-02, -2.3026e-02,\n",
       "                      -2.6207e-03, -1.1436e-02, -1.2794e-02,  5.7983e-02,  1.9646e-03,\n",
       "                      -5.0011e-03, -3.2135e-02,  6.1913e-03, -2.4719e-02,  1.7105e-02,\n",
       "                      -5.4436e-03,  2.4307e-02,  3.6049e-03,  3.4847e-03, -5.4810e-02,\n",
       "                       7.2289e-03, -4.4159e-02,  2.9816e-02, -1.1224e-01, -1.9617e-01,\n",
       "                       1.0551e-02,  1.7349e-02, -2.8183e-02,  1.8330e-03,  1.4343e-02,\n",
       "                       1.0368e-02,  2.4155e-02, -2.0947e-01,  3.2257e-02, -3.7079e-03,\n",
       "                      -4.5357e-03,  4.0405e-02, -1.9760e-02,  1.6922e-02, -3.0899e-02,\n",
       "                       2.4551e-02,  1.6693e-02, -8.9188e-03,  2.2156e-02,  7.8278e-03,\n",
       "                      -1.9638e-02,  1.1024e-03, -1.3313e-02, -2.9327e-02,  9.6970e-03,\n",
       "                       1.4755e-02, -6.3477e-03,  3.0251e-03, -1.2405e-02, -1.3191e-02,\n",
       "                       4.8027e-03,  2.4582e-02,  2.8534e-02,  9.8267e-03, -1.1151e-01,\n",
       "                      -1.2197e-03, -7.9269e-03, -4.3610e-02, -2.7039e-02, -2.2681e-01,\n",
       "                       1.8103e-01, -5.0018e-02, -1.1627e-02,  3.4521e-01,  9.1370e-02,\n",
       "                       8.4534e-02,  4.8065e-02, -8.2715e-01,  4.3893e-04, -1.3147e-01,\n",
       "                       3.8422e-02,  2.7148e-01,  1.4758e-01,  1.4783e-01,  7.1045e-02,\n",
       "                      -1.9299e-01,  5.8014e-02, -5.0507e-03, -5.6488e-02,  3.0151e-01,\n",
       "                      -1.9128e-01, -1.5039e-01, -7.5378e-02,  1.0002e-02,  1.2048e-01,\n",
       "                       1.1957e-01, -1.2079e-01,  2.7563e-01,  5.3101e-02, -2.6758e-01,\n",
       "                       9.8022e-02, -3.0884e-02, -9.6863e-02, -6.9458e-02, -1.7960e-02,\n",
       "                      -2.2168e-01, -9.2712e-02,  1.1383e-01, -2.7490e-01,  7.7026e-02,\n",
       "                       3.0713e-01,  1.2207e-02,  5.9082e-02, -5.7068e-02, -4.6310e-03,\n",
       "                       7.0572e-03, -1.7136e-02, -7.4158e-03, -2.5000e-01, -4.3286e-01,\n",
       "                      -1.5688e-03,  2.2305e+00,  2.5040e-02,  6.8245e-03,  3.9526e-01,\n",
       "                      -8.4229e-02, -1.0974e-01, -1.0663e-01, -1.0162e-01, -2.3364e-01,\n",
       "                      -2.8442e-02, -4.4165e-01, -3.8794e-01, -1.8242e+00, -4.6484e+00,\n",
       "                      -2.7295e-01,  2.0959e-01, -1.1221e+00,  6.7969e-01, -8.6365e-02,\n",
       "                       3.2544e-01,  2.8857e-01,  3.9160e-01,  6.0400e-01,  1.6919e-01,\n",
       "                      -6.6357e-01, -6.3574e-01,  4.4067e-01,  1.3647e-01,  5.2490e-01,\n",
       "                      -9.8828e-01, -2.1985e-01,  4.8071e-01, -3.9917e-01,  4.0845e-01,\n",
       "                       9.2871e-01, -7.8223e-01, -4.7925e-01,  6.2256e-01, -8.9294e-02,\n",
       "                      -1.4221e-01, -4.0381e-01,  2.6245e-01, -2.5781e-01, -1.7773e-01,\n",
       "                       2.8174e-01,  4.8071e-01, -7.3792e-02, -2.0862e-01, -3.8110e-01,\n",
       "                       1.5625e-01,  7.7783e-01, -1.6284e-01, -6.4551e-01, -1.7422e+00,\n",
       "                      -2.4246e-02, -2.0007e-01, -8.2715e-01, -2.6489e-01, -2.5562e-01,\n",
       "                      -1.1113e+00,  6.1572e-01,  7.3120e-02,  1.7666e+00,  2.1250e+00,\n",
       "                       6.6504e-01,  5.4248e-01,  4.4873e-01,  4.6436e-01,  1.5060e-02,\n",
       "                      -2.6306e-02, -4.0186e-01, -2.0068e-01, -7.2119e-01, -8.3435e-02,\n",
       "                      -3.7804e-03, -2.5360e-02,  1.7105e-02,  2.0752e-03, -1.3885e-02,\n",
       "                       2.3453e-02,  1.3542e-02, -1.2192e-02, -5.6305e-03, -3.4161e-03,\n",
       "                       1.1740e-03, -1.0414e-02, -5.8022e-03, -9.9030e-03,  2.8366e-02,\n",
       "                      -1.0475e-02,  1.1345e-02, -2.0172e-02, -3.7201e-02, -2.0935e-02,\n",
       "                      -1.1711e-03,  1.0994e-02, -2.0325e-02, -1.0208e-02, -2.3079e-03,\n",
       "                       4.9744e-02,  5.5115e-02,  4.5471e-03, -4.6967e-02,  4.6753e-02,\n",
       "                       7.7057e-03,  4.9095e-03,  1.7490e-03, -1.8835e-04,  4.9820e-03,\n",
       "                      -1.3557e-02, -1.8051e-02, -1.8997e-02,  2.7344e-02,  2.9480e-02,\n",
       "                       3.5152e-03, -4.6005e-03, -5.9891e-03, -1.7595e-03, -7.9041e-03,\n",
       "                      -1.1436e-02,  2.9907e-02, -1.5354e-03, -8.6498e-04, -1.5602e-02,\n",
       "                       2.7054e-02,  1.8129e-03, -6.8321e-03, -2.5589e-02, -9.0698e-02,\n",
       "                       1.7071e-03,  1.1200e-02, -2.3605e-02, -1.5549e-02, -4.7577e-02,\n",
       "                      -1.5839e-02,  9.4910e-03,  2.3758e-02,  1.7044e-02, -9.1705e-03,\n",
       "                       1.2927e-01,  5.1270e-01, -7.0605e-01,  6.0791e-02,  3.5791e-01,\n",
       "                       9.8206e-02, -1.0205e-01,  7.6965e-02, -3.4119e-02,  1.7969e-01,\n",
       "                      -8.5327e-02, -5.8365e-03,  4.0466e-02, -3.1885e-01,  2.4353e-02,\n",
       "                      -1.0187e-01, -1.1066e-01,  1.6565e-01, -4.6906e-02,  6.7993e-02,\n",
       "                       7.4463e-02,  2.0544e-01,  1.0605e+00, -6.6162e-02,  4.7791e-02,\n",
       "                       5.5273e-01,  2.2430e-02,  9.3628e-02,  4.9042e-02, -2.2937e-01,\n",
       "                       2.9327e-02, -1.3513e-01, -2.7222e-02, -1.3660e-01, -2.5659e-01,\n",
       "                      -7.2632e-02, -5.9082e-02, -3.6621e-01,  1.8799e-02,  3.8776e-03,\n",
       "                      -2.5952e-01, -1.7493e-01, -1.4450e-02,  2.2266e-01,  3.3264e-03,\n",
       "                       1.6582e+00,  3.1738e-02, -1.5149e-01, -8.3105e-01,  4.0955e-02,\n",
       "                       3.1665e-01,  4.4434e-01,  1.5566e+00, -1.8958e-01, -8.7830e-02,\n",
       "                      -3.9597e-03, -1.3721e-01, -2.3071e-02,  1.8140e-01,  9.4238e-02,\n",
       "                       3.0518e-01,  1.6089e-01, -3.5187e-02, -5.4443e-01, -3.5181e-01,\n",
       "                       6.9336e-02,  7.4707e-02, -3.3374e-01, -2.6641e+00, -1.0869e+00,\n",
       "                      -4.1107e-02, -1.2024e-01,  2.7271e-01, -3.7354e-01, -1.7871e+00,\n",
       "                      -8.7976e-04,  6.0498e-01,  6.8311e-01,  9.5020e-01, -1.0459e+00,\n",
       "                      -6.2988e-01, -8.9783e-02, -2.4939e-01,  4.8901e-01, -9.3231e-03,\n",
       "                      -1.4688e+00,  2.5195e-01, -4.2999e-02, -4.2038e-03, -1.2549e+00,\n",
       "                      -1.0742e+00, -8.1982e-01,  1.5352e+00,  6.9922e-01,  5.3320e-01,\n",
       "                       5.5078e-01, -7.0752e-01,  5.4932e-01,  7.6807e-01,  1.2803e+00,\n",
       "                       1.0420e+00,  9.5508e-01,  7.4402e-02,  1.1383e-02,  2.1042e-02,\n",
       "                       1.9023e+00, -3.7402e-01, -5.0049e-02, -2.5220e-01,  1.6104e+00,\n",
       "                      -5.0635e-01, -9.5898e-01, -1.0420e+00, -6.4697e-01, -2.5708e-01,\n",
       "                       8.7354e-01,  1.3477e+00,  1.0293e+00,  2.3059e-01, -1.1807e+00,\n",
       "                       5.1709e-01, -7.1594e-02,  1.0132e-01, -4.6436e-01,  5.2246e-01,\n",
       "                       3.9282e-01,  5.6335e-02,  7.2571e-02, -2.2964e-02,  3.6957e-02,\n",
       "                       1.1009e-02,  1.7563e-02, -4.8332e-03,  1.4328e-02,  3.0518e-02,\n",
       "                      -3.1891e-02,  7.8888e-03, -4.6295e-02, -5.0293e-02,  4.0703e-03,\n",
       "                      -1.2901e-02, -1.0605e-02, -2.9327e-02,  6.7139e-03,  6.6948e-03,\n",
       "                       3.0350e-02, -8.2321e-03, -1.1883e-03,  4.1962e-02,  3.8483e-02,\n",
       "                      -4.7058e-02, -5.2309e-04, -3.3264e-02,  1.2039e-02,  7.3013e-03,\n",
       "                       1.8280e-02, -1.4257e-03,  1.4374e-02, -2.1637e-02, -9.0103e-03,\n",
       "                       8.9294e-02,  1.0004e-03,  1.4153e-02,  3.7085e-01, -5.9891e-03,\n",
       "                       5.1361e-02, -5.1918e-03, -3.4424e-02, -3.8055e-02, -8.6288e-03,\n",
       "                      -9.6054e-03,  7.3471e-03,  6.8932e-03, -1.8444e-03, -5.9586e-03,\n",
       "                      -3.7292e-02, -3.8910e-03, -2.9541e-02,  1.5472e-02, -1.7883e-02,\n",
       "                      -3.5725e-03, -1.7456e-02, -5.6763e-02,  3.9703e-02,  1.7990e-02,\n",
       "                       3.3112e-02,  1.8387e-02, -4.1595e-02,  2.1194e-02, -9.1629e-03,\n",
       "                      -2.9968e-02, -4.7998e-01,  1.1835e-01, -1.8726e-01,  2.0340e-02,\n",
       "                       2.9346e-01, -8.5144e-02, -4.0436e-02, -1.0815e-01,  4.7546e-02,\n",
       "                       9.3002e-03, -1.9958e-01,  1.2445e-01, -9.7229e-02, -3.8721e-01,\n",
       "                       4.6310e-03, -6.2943e-03,  1.8542e-01, -4.1016e-02,  2.1240e-01,\n",
       "                       6.5063e-02, -1.2292e-01, -9.9487e-02,  1.6914e+00,  7.7332e-02,\n",
       "                      -8.3862e-02,  6.8481e-02, -3.8818e-01, -2.5366e-01, -8.3191e-02,\n",
       "                       7.6172e-02,  2.4219e+00,  8.4961e-02,  5.2393e-01, -3.0869e-02,\n",
       "                       3.9575e-01, -1.1523e-01,  1.6296e-01, -1.9604e-01,  1.4168e-02,\n",
       "                       6.2744e-02,  8.2642e-02,  8.7128e-03, -1.1700e-01, -6.6040e-02,\n",
       "                      -1.9995e-01, -2.1948e-01,  9.7900e-02,  1.3501e-01,  2.5806e-01,\n",
       "                      -7.9285e-02,  2.2046e-01, -1.9202e-01, -1.1566e-01,  1.3806e-01,\n",
       "                      -1.6098e-02,  6.6650e-02, -6.2828e-03,  9.4223e-03,  9.2651e-02,\n",
       "                       2.0911e-01, -2.5317e-01,  1.3013e-01, -5.2686e-01,  1.0620e-02,\n",
       "                       1.1904e+00,  8.8928e-02,  1.5391e+00, -5.5762e-01, -4.7656e-01,\n",
       "                       8.6133e-01,  6.8213e-01, -7.6465e-01,  1.0898e+00,  4.3555e-01,\n",
       "                       2.4863e+00,  3.4180e-01,  3.1372e-01,  1.8125e+00, -1.2024e-01,\n",
       "                      -4.6509e-01, -6.4111e-01,  1.0059e-01, -2.9727e+00, -4.0845e-01,\n",
       "                      -1.0215e+00,  1.1123e+00, -9.9609e+00,  8.4473e-01, -9.3213e-01,\n",
       "                       5.1709e-01,  1.0371e+00,  2.3965e+00,  6.4795e-01, -1.6973e+00,\n",
       "                      -1.3242e+01, -1.1035e+00, -4.7539e+00, -1.1416e+00, -2.4824e+00,\n",
       "                       1.6199e-01, -1.8516e+00,  1.1992e+00,  1.1230e+00,  1.2217e+00,\n",
       "                       1.0732e+00, -7.0752e-01, -8.2812e-01,  1.2383e+00,  8.0127e-01,\n",
       "                       1.3223e+00,  7.3914e-02,  7.7002e-01, -2.4277e+00, -5.9473e-01,\n",
       "                      -1.0908e+00,  2.9663e-01, -3.1836e-01, -2.3594e+00, -1.0889e+00,\n",
       "                       4.3359e-01, -1.8721e+00,  8.9990e-01, -2.3511e-01, -2.4902e+00,\n",
       "                       7.8076e-01, -1.3340e+00,  2.2969e+00,  4.5288e-01,  6.9122e-03,\n",
       "                      -1.6418e-02,  1.6327e-02, -4.2343e-03,  8.7585e-03, -3.7048e-02,\n",
       "                      -1.5305e-02, -4.2629e-04, -2.5162e-02, -1.5823e-02,  7.4387e-03,\n",
       "                       1.7960e-02,  4.1504e-02, -6.3019e-03,  6.6948e-03,  2.7954e-02,\n",
       "                      -2.5726e-02, -2.1301e-02, -3.3512e-03, -4.1122e-03,  4.3243e-02,\n",
       "                      -3.6430e-03, -6.8092e-03, -1.1757e-02, -1.9028e-02,  2.5909e-02,\n",
       "                       8.4457e-03,  3.3142e-02, -2.0554e-02,  9.6083e-04, -1.7365e-02,\n",
       "                      -3.0807e-02,  1.2093e-02, -2.0966e-02, -1.3977e-02, -2.2812e-02,\n",
       "                      -6.0303e-02,  8.1482e-02,  1.0483e-02,  5.2307e-02,  1.2291e-02,\n",
       "                      -7.5012e-02, -2.2602e-03,  2.0432e-02,  1.2512e-03,  1.4626e-02,\n",
       "                       2.0691e-02, -2.0111e-02, -8.5297e-03,  1.3062e-02, -5.5664e-02,\n",
       "                       1.0333e-01, -6.8779e-03,  1.5129e-02,  4.0627e-03,  2.1271e-02,\n",
       "                      -2.0523e-02,  2.8275e-02,  3.5553e-02,  5.4092e-03,  1.9665e-03,\n",
       "                      -8.9645e-03, -3.4851e-02,  4.9927e-02])),\n",
       "             ('transformer.h.12.self_attention.dense.weight',\n",
       "              tensor([[ 1.2306e-02, -9.9564e-04,  1.6129e-02,  ..., -2.1637e-02,\n",
       "                        1.1650e-02,  6.9284e-04],\n",
       "                      [-2.3621e-02,  9.2773e-03,  3.3894e-03,  ..., -1.1200e-02,\n",
       "                        8.1177e-03,  4.1443e-02],\n",
       "                      [ 1.8356e-02, -2.6436e-03,  5.9204e-03,  ...,  1.1986e-02,\n",
       "                        1.9760e-02, -1.0361e-02],\n",
       "                      ...,\n",
       "                      [-2.9861e-02, -1.4893e-02, -1.7700e-02,  ..., -9.2220e-04,\n",
       "                       -1.1978e-02, -3.5217e-02],\n",
       "                      [ 5.0087e-03, -3.2997e-03,  2.7084e-02,  ...,  1.5190e-02,\n",
       "                       -9.3937e-05, -1.2314e-02],\n",
       "                      [ 5.8403e-03, -1.2379e-03, -1.5884e-02,  ...,  3.4847e-03,\n",
       "                        2.8152e-02, -1.9653e-02]])),\n",
       "             ('transformer.h.12.self_attention.dense.bias',\n",
       "              tensor([ 0.0122,  0.0222,  0.0343,  ...,  0.0076, -0.0850,  0.0303])),\n",
       "             ('transformer.h.12.post_attention_layernorm.weight',\n",
       "              tensor([1.1816, 1.3750, 1.0762,  ..., 1.4170, 1.5244, 0.8506])),\n",
       "             ('transformer.h.12.post_attention_layernorm.bias',\n",
       "              tensor([ 0.0589,  0.0249,  0.1339,  ..., -0.0187,  0.0552,  0.0701])),\n",
       "             ('transformer.h.12.mlp.dense_h_to_4h.weight',\n",
       "              tensor([[-0.0274,  0.0184, -0.0130,  ..., -0.0041,  0.0392,  0.0279],\n",
       "                      [ 0.0298, -0.0141, -0.0049,  ...,  0.0189, -0.0203,  0.0325],\n",
       "                      [ 0.0087, -0.0165,  0.0368,  ...,  0.0031,  0.0035,  0.0062],\n",
       "                      ...,\n",
       "                      [ 0.0028, -0.0117,  0.0356,  ...,  0.0267, -0.0080,  0.0177],\n",
       "                      [ 0.0257, -0.0253, -0.0247,  ..., -0.0089, -0.0152, -0.0076],\n",
       "                      [ 0.0060,  0.0166, -0.0034,  ...,  0.0086,  0.0112, -0.0081]])),\n",
       "             ('transformer.h.12.mlp.dense_h_to_4h.bias',\n",
       "              tensor([-0.4934, -0.5767, -0.5483,  ..., -0.3865, -0.0412,  0.0143])),\n",
       "             ('transformer.h.12.mlp.dense_4h_to_h.weight',\n",
       "              tensor([[-8.0824e-04,  1.5190e-02,  8.6975e-03,  ..., -3.5133e-03,\n",
       "                       -1.4734e-03, -1.4252e-02],\n",
       "                      [-4.2877e-03,  2.7504e-03, -6.4468e-04,  ..., -3.1097e-02,\n",
       "                        1.3649e-02, -9.4604e-03],\n",
       "                      [-7.1411e-03, -5.1918e-03,  5.8594e-03,  ...,  1.0475e-02,\n",
       "                        1.9112e-03, -8.7357e-03],\n",
       "                      ...,\n",
       "                      [-3.2471e-02,  4.5681e-04, -3.4485e-02,  ...,  9.9599e-05,\n",
       "                       -1.8890e-02, -8.8501e-03],\n",
       "                      [ 5.4779e-02, -1.6861e-02,  4.8161e-04,  ...,  3.3783e-02,\n",
       "                        1.6754e-02, -1.3222e-02],\n",
       "                      [ 1.5900e-02,  1.3138e-02, -1.1663e-03,  ...,  1.2955e-02,\n",
       "                        1.4420e-03,  1.4999e-02]])),\n",
       "             ('transformer.h.12.mlp.dense_4h_to_h.bias',\n",
       "              tensor([ 0.0693, -0.1179, -0.0938,  ..., -0.2120,  0.2244,  0.0019])),\n",
       "             ('transformer.h.13.input_layernorm.weight',\n",
       "              tensor([1.3555, 1.7842, 1.2080,  ..., 1.9678, 1.8652, 0.9595])),\n",
       "             ('transformer.h.13.input_layernorm.bias',\n",
       "              tensor([ 0.0161,  0.0467,  0.0490,  ...,  0.1405,  0.0504, -0.0216])),\n",
       "             ('transformer.h.13.self_attention.query_key_value.weight',\n",
       "              tensor([[ 0.0129,  0.0246, -0.0053,  ...,  0.0387,  0.0346,  0.0157],\n",
       "                      [-0.0284, -0.0086,  0.0306,  ...,  0.0253, -0.0117,  0.0119],\n",
       "                      [ 0.0155,  0.0206,  0.0334,  ...,  0.0197, -0.0006, -0.0020],\n",
       "                      ...,\n",
       "                      [-0.0157, -0.0454, -0.0055,  ...,  0.0125,  0.0301,  0.0194],\n",
       "                      [ 0.0119, -0.0056, -0.0036,  ..., -0.0073, -0.0069,  0.0077],\n",
       "                      [-0.0197, -0.0058, -0.0066,  ...,  0.0127, -0.0159, -0.0005]])),\n",
       "             ('transformer.h.13.self_attention.query_key_value.bias',\n",
       "              tensor([ 1.6394e-01,  4.8035e-02, -1.8884e-01, -2.5000e-01, -2.3840e-01,\n",
       "                       7.0801e-02,  5.7129e-01, -3.1543e-01, -2.2675e-02, -6.2793e-01,\n",
       "                       1.0449e-01,  2.1289e-01,  1.3196e-01,  4.4604e-01,  2.2363e-01,\n",
       "                       3.6011e-01,  5.7922e-02, -5.0995e-02,  7.3669e-02, -4.8804e-01,\n",
       "                      -7.9041e-02,  1.6484e+00,  1.0791e-01, -5.2399e-02, -4.9390e-01,\n",
       "                      -2.2229e-01, -6.1737e-02,  4.2603e-01,  2.5098e+00,  3.5913e-01,\n",
       "                      -2.5854e-01, -2.2839e-01,  1.3477e-01, -1.7822e-01,  4.1333e-01,\n",
       "                       5.0201e-02,  6.9922e-01,  2.6953e-01, -2.7295e-01, -2.2949e-01,\n",
       "                      -1.6418e-01, -3.1177e-01,  1.6769e-02,  4.1284e-01,  2.7686e-01,\n",
       "                       3.4497e-01, -1.2236e+00,  7.2136e-03, -4.5630e-01, -1.6699e-01,\n",
       "                      -1.3416e-01, -3.7048e-02, -2.8369e-01, -7.3547e-02, -3.6353e-01,\n",
       "                       1.7749e-01, -5.4590e-01,  4.8523e-02, -6.8506e-01,  1.8628e-01,\n",
       "                       1.1177e-03, -3.6792e-01, -2.3523e-01, -8.7280e-03,  1.9348e-02,\n",
       "                      -1.1902e-02,  1.8787e-03,  8.5449e-03, -1.6525e-02,  9.0408e-03,\n",
       "                       1.5396e-02, -1.0765e-02,  3.3512e-03, -2.9831e-02, -6.4880e-02,\n",
       "                      -2.8458e-02, -1.8036e-02, -2.7466e-02, -7.9203e-04,  1.0635e-02,\n",
       "                      -1.5762e-02,  3.8422e-02,  1.2909e-02,  8.2016e-03,  1.6756e-03,\n",
       "                      -4.8315e-01,  5.6061e-02,  2.1027e-02, -8.0490e-03,  1.5535e-03,\n",
       "                      -2.0493e-02, -2.6291e-02, -2.0706e-02,  5.9166e-03, -2.4719e-02,\n",
       "                      -4.3976e-02,  4.2786e-02, -3.4637e-03, -7.7026e-02, -5.1910e-02,\n",
       "                       2.2842e-02, -2.8133e-04, -2.6154e-02,  8.0795e-03, -5.5206e-02,\n",
       "                       4.3335e-02, -2.0111e-02,  3.2715e-02, -1.5579e-02, -4.9835e-02,\n",
       "                       2.0569e-02, -5.0087e-03, -2.8717e-02,  2.6733e-02,  2.5864e-03,\n",
       "                      -2.9465e-02, -3.0197e-02, -3.1952e-02, -4.1046e-02, -2.9282e-02,\n",
       "                      -5.7953e-02,  6.5231e-03,  2.0828e-02, -3.4729e-02,  3.3752e-02,\n",
       "                       1.3252e-02,  8.3618e-02,  9.9182e-03,  7.8659e-03, -3.0746e-02,\n",
       "                       8.3542e-03, -2.7695e-02, -2.7100e-01, -1.3092e-02, -8.4412e-02,\n",
       "                      -7.4501e-03, -2.2339e-02, -2.6611e-02, -2.8915e-02,  1.9287e-02,\n",
       "                      -1.0826e-02, -1.3481e-02, -3.2410e-02, -2.1240e-02,  2.9831e-02,\n",
       "                      -3.0197e-02, -3.7689e-02,  2.4551e-02,  1.9730e-02,  1.4832e-02,\n",
       "                      -2.2675e-02,  1.7136e-02, -3.3417e-02, -3.4451e-04,  2.7008e-02,\n",
       "                      -2.4033e-02, -4.9820e-03, -1.2688e-02, -2.1255e-02,  1.7517e-02,\n",
       "                       1.6739e-02, -4.8447e-03, -5.1056e-02,  7.4890e-02,  8.1100e-03,\n",
       "                       1.9026e-03,  4.2023e-02,  1.3933e-03, -1.5312e-02,  2.3376e-02,\n",
       "                      -1.4656e-02,  3.5980e-02,  1.5030e-02, -1.0577e-01,  1.4488e-02,\n",
       "                       6.9618e-03,  6.0349e-03,  8.8959e-03, -4.3579e-02, -7.0068e-02,\n",
       "                       9.6680e-02, -3.2768e-03, -3.4065e-03,  2.8992e-03, -4.7485e-02,\n",
       "                       1.6724e-02,  1.2177e-02,  3.2745e-02, -3.1586e-02, -1.0445e-02,\n",
       "                       2.2980e-02,  1.3557e-02,  2.0332e+00, -2.1289e-01, -7.6965e-02,\n",
       "                       1.1139e-01,  6.8787e-02,  1.1665e-02,  1.7737e-01,  2.7979e-01,\n",
       "                      -1.0815e-01, -1.1123e+00,  2.1753e-01,  3.7933e-02,  2.1545e-01,\n",
       "                      -2.1692e-01,  1.1627e-01,  5.0732e-01,  8.7891e-02,  4.0332e-01,\n",
       "                       1.8005e-02, -3.0762e-01, -2.8735e-01,  1.1230e-01, -2.3572e-01,\n",
       "                      -1.1023e-01, -2.5635e-01,  1.8982e-01, -1.0822e-01,  3.5675e-02,\n",
       "                      -4.2389e-02, -6.4087e-02,  5.2521e-02, -2.1680e-01, -1.6455e-01,\n",
       "                      -2.3022e-01,  2.1423e-01, -4.1321e-02,  1.8219e-02,  1.8396e-01,\n",
       "                      -4.6814e-02, -1.3770e-01, -1.1078e-02, -1.3525e-01, -3.3356e-02,\n",
       "                      -5.0964e-02, -1.4404e-01,  1.7676e-01, -1.1975e-01, -1.2793e-01,\n",
       "                       7.9102e-01,  5.0720e-02,  1.6199e-01, -1.2115e-01,  1.7746e-02,\n",
       "                      -1.4844e-01, -1.3831e-01, -4.0308e-01, -1.1523e+00,  5.7251e-02,\n",
       "                      -2.1545e-02, -1.8402e-02, -6.1035e-01, -2.3779e-01,  6.1218e-02,\n",
       "                       1.6260e-01,  4.8071e-01, -8.4863e-01, -4.8706e-01, -1.9434e-01,\n",
       "                       1.3262e+00,  2.6001e-01, -6.5088e-01, -1.3013e-01,  4.2456e-01,\n",
       "                       2.3608e-01, -3.6450e-01,  1.0907e-01,  2.2974e-01, -3.5840e-01,\n",
       "                       1.0957e+00,  9.1504e-01,  6.4355e-01,  4.9048e-01, -2.4316e-01,\n",
       "                      -1.3123e-01, -6.0303e-01,  3.0508e+00, -6.0156e-01,  6.7787e-03,\n",
       "                       6.1670e-01, -9.2676e-01,  3.4210e-02, -2.9449e-02, -2.7686e-01,\n",
       "                      -2.8345e-01,  3.4155e-01,  8.5510e-02, -6.1670e-01,  1.3008e+00,\n",
       "                       3.7866e-01, -2.4109e-01,  1.7078e-01,  5.3802e-02,  1.2244e-01,\n",
       "                       2.8418e-01,  5.5518e-01, -1.2686e+00, -3.7939e-01, -1.4929e-01,\n",
       "                      -4.5820e+00, -6.3135e-01, -2.6025e-01, -5.0293e-01,  7.7344e-01,\n",
       "                      -5.9863e-01,  6.1572e-01,  3.2617e-01, -7.7930e-01, -2.0645e+00,\n",
       "                       4.7729e-01, -7.6611e-01, -5.5762e-01,  7.8125e-01,  2.5903e-01,\n",
       "                      -6.9580e-02, -3.2129e-01,  9.9365e-01, -3.4766e-01, -5.5127e-01,\n",
       "                      -2.9980e-01,  1.9238e-01,  3.4698e-02, -1.9141e+00, -1.6724e-01,\n",
       "                      -1.4368e-01,  5.8014e-02,  1.1810e-02,  2.3853e-01, -1.2891e-01,\n",
       "                      -2.4734e-02,  2.4707e-01, -3.9032e-02,  1.4259e-02, -5.5634e-02,\n",
       "                       5.7343e-02,  2.4255e-01, -1.5234e-01,  1.1115e-01, -4.4116e-01,\n",
       "                       3.3203e-02, -1.5552e-01,  2.5009e-02, -3.4448e-01,  7.1472e-02,\n",
       "                       2.4139e-02, -1.9485e-02, -2.5040e-02,  8.9294e-02, -3.6963e-01,\n",
       "                       6.1218e-02,  2.0776e-01,  3.6285e-02, -1.4587e-01, -4.2749e-01,\n",
       "                       1.0004e-01, -5.4207e-03,  2.6147e-01,  1.0590e-02,  6.0883e-02,\n",
       "                       2.1814e-01, -1.4832e-01,  4.1290e-02,  1.5053e-02, -6.9092e-02,\n",
       "                       1.9141e-01, -6.7090e-01,  4.3457e-02, -1.4722e-01, -1.7871e+00,\n",
       "                       4.6021e-02, -2.0703e-01, -1.8884e-01, -7.1594e-02,  6.5613e-02,\n",
       "                       2.1472e-01,  1.9250e-01,  1.3000e-01,  3.7598e-02, -1.5015e-01,\n",
       "                      -8.5754e-02,  2.4307e-02,  1.3354e-01,  5.7831e-03,  8.0200e-02,\n",
       "                       9.9170e-01,  5.8472e-02, -1.0718e-01, -6.4746e-01, -2.7124e-01,\n",
       "                      -2.5854e-01,  1.0244e+00,  7.6866e-03, -2.3657e-01,  6.6357e-01,\n",
       "                      -1.2021e+00, -7.0898e-01, -2.4084e-01,  9.7363e-01,  1.8042e-01,\n",
       "                      -1.5967e-01,  1.4131e+00,  2.5024e-01, -3.8037e-01, -9.5264e-01,\n",
       "                       3.1396e-01, -8.5205e-01, -4.4189e-01, -8.8959e-03,  3.6328e-01,\n",
       "                      -2.0251e-01,  1.4502e+00,  2.3462e-01, -6.0986e-01,  4.9805e-01,\n",
       "                       7.0459e-01,  1.1377e+00,  4.5435e-01, -1.0264e+00,  1.1536e-01,\n",
       "                      -2.3727e-02, -3.2373e-01,  9.1309e-01, -5.8301e-01, -2.7539e-01,\n",
       "                       2.3804e-01, -3.4302e-01,  2.0137e+00, -2.3792e-01,  9.6240e-01,\n",
       "                       4.5728e-01, -5.3223e-01,  7.9150e-01, -2.7871e+00,  1.6641e+00,\n",
       "                       7.1143e-01,  9.2896e-02, -2.4695e-01, -1.6406e-01,  2.7393e-01,\n",
       "                      -7.7539e-01, -1.0039e+00,  9.2773e-01,  3.7109e-01, -1.1992e+00,\n",
       "                      -4.4800e-01,  1.1787e+00, -5.8984e-01,  3.8848e+00,  1.2211e+01,\n",
       "                      -2.8086e+00,  1.0645e+00, -1.0383e+01, -4.8906e+00, -7.3398e+00,\n",
       "                       1.5898e+01,  2.5371e+00, -1.7969e+00,  4.6055e+00, -9.6562e+00,\n",
       "                      -9.0703e+00, -6.5703e+00,  1.1359e+01,  2.2156e-02, -8.1836e-01,\n",
       "                       2.2062e+01,  9.0781e+00, -2.0918e+00, -1.0484e+01,  3.9902e+00,\n",
       "                      -1.0148e+01, -7.1914e+00, -6.7529e-01,  8.0938e+00, -1.8340e+00,\n",
       "                       1.8422e+01,  4.5898e+00, -9.3828e+00,  6.3516e+00,  5.6602e+00,\n",
       "                       1.4992e+01,  5.1953e+00, -9.1172e+00,  3.8867e+00, -4.2227e+00,\n",
       "                      -5.5820e+00,  9.4688e+00, -9.2422e+00,  3.8989e-01, -3.6475e-01,\n",
       "                      -5.7227e+00,  2.4453e+01, -3.4590e+00,  1.3461e+01,  6.4102e+00,\n",
       "                      -8.7188e+00,  9.3281e+00, -3.4875e+01,  2.2484e+01,  8.7969e+00,\n",
       "                       3.5781e+00, -2.6289e+00, -4.6289e+00,  2.8867e+00, -1.2992e+01,\n",
       "                      -8.9609e+00,  1.1445e+01,  3.1152e+00, -1.2906e+01, -9.6016e+00,\n",
       "                       1.2273e+01, -2.2773e+00, -1.1963e-02,  7.3967e-03, -1.4885e-02,\n",
       "                       3.6469e-02, -1.0857e-02, -2.3376e-02, -1.4534e-02, -4.8141e-03,\n",
       "                       2.9022e-02,  2.5696e-02, -1.9882e-02,  2.8992e-03, -9.4452e-03,\n",
       "                      -3.0426e-02,  2.8839e-02, -2.1973e-02, -1.6220e-02,  1.0788e-02,\n",
       "                      -5.6824e-02,  5.7495e-02,  2.5177e-03, -1.5686e-02,  1.1871e-02,\n",
       "                      -8.0750e-02,  2.9144e-03, -3.4271e-02, -1.4206e-02,  1.8053e-03,\n",
       "                      -2.2964e-02,  8.9233e-02,  6.5947e-04, -1.4503e-02,  9.5520e-03,\n",
       "                      -2.6489e-02,  8.4076e-03, -3.1986e-03,  1.9043e-02, -3.3295e-02,\n",
       "                       3.5381e-03, -2.0508e-02, -2.8753e-04,  1.8555e-02,  7.0305e-03,\n",
       "                      -6.3972e-03, -3.4088e-02,  3.4657e-03, -3.7994e-02,  3.5334e-04,\n",
       "                      -3.9711e-03,  2.7633e-04,  6.5422e-03, -1.9821e-02,  2.9312e-02,\n",
       "                      -2.6093e-02,  4.8866e-03,  4.0512e-03,  7.0343e-03, -5.7983e-04,\n",
       "                       1.0727e-02, -7.0007e-02, -7.1869e-03, -1.0208e-02,  2.0239e-01,\n",
       "                      -3.6896e-02,  1.7031e+00,  9.8755e-02, -2.0176e+00, -1.5205e+00,\n",
       "                       9.3311e-01,  7.7539e-01, -4.5654e-01, -8.4131e-01,  1.1006e+00,\n",
       "                       7.0996e-01, -2.5342e-01,  1.5891e+01, -2.8638e-01,  2.4277e+00,\n",
       "                      -1.1172e+00,  1.2061e+00, -7.1143e-01, -5.3564e-01,  4.0161e-02,\n",
       "                      -5.0195e-01,  8.7891e-01, -8.6914e-01,  1.2588e+00,  1.5664e+00,\n",
       "                       1.4575e-01, -3.2949e+00, -2.6191e+00, -1.3125e+00,  2.6099e-01,\n",
       "                      -1.3486e+00,  2.0618e-01, -9.0469e+00,  2.3438e+00,  1.1113e+00,\n",
       "                       4.2847e-02, -1.3789e+00, -9.6802e-02,  1.2881e+00, -1.2285e+00,\n",
       "                       4.9585e-01,  1.8340e+00,  1.9861e-01,  3.4082e+00,  1.1455e+00,\n",
       "                      -6.3721e-01,  2.5352e+00, -2.1992e+00,  1.3301e+00,  2.8340e+00,\n",
       "                      -1.2920e+00,  1.6078e+01,  2.9419e-01, -1.3643e+00,  6.6699e-01,\n",
       "                       2.6953e-01, -3.5309e-02,  6.7078e-02, -8.7793e-01,  2.2246e+00,\n",
       "                      -1.4844e-01, -1.4346e+00, -1.4639e+00, -1.2708e-01,  8.3252e-01,\n",
       "                      -5.8154e-01,  2.2695e+00,  8.2910e-01, -2.7734e+00,  3.6768e-01,\n",
       "                      -7.3682e-01, -9.9756e-01,  1.0312e+01,  4.6758e+00, -6.3008e+00,\n",
       "                       1.7432e+00,  3.7480e+00, -3.1758e+00, -1.9580e+00, -1.5361e+00,\n",
       "                       4.1602e+00, -2.4707e+00,  1.4922e+00,  6.9629e-01,  1.3682e+00,\n",
       "                      -7.1172e+00,  2.0349e-01,  1.0244e+00, -3.1270e+00, -5.1758e-01,\n",
       "                      -3.8633e+00,  2.8719e+01, -1.7969e+01,  1.9092e+00, -6.5117e+00,\n",
       "                       6.0059e-01, -3.0615e-01, -2.3770e+00, -7.5234e+00,  1.9958e-01,\n",
       "                       2.8281e+00,  5.3867e+00, -2.0137e+00,  3.2422e+00,  1.7861e+00,\n",
       "                      -6.7109e+00, -3.3066e+00,  4.8169e-01,  8.6768e-01,  1.3574e-01,\n",
       "                       3.5522e-01,  3.6621e-01,  6.7090e-01,  9.1602e-01,  3.3711e+00,\n",
       "                      -2.0117e+00,  1.0830e+00,  3.2690e-01,  7.4121e-01, -6.6553e-01,\n",
       "                      -2.9766e+00, -8.1562e+00,  2.5171e-01,  8.1836e-01,  1.8535e+00,\n",
       "                       2.9028e-01,  1.0117e+00,  5.3594e+00, -1.9043e+00, -4.5319e-02,\n",
       "                      -3.3020e-02, -7.9895e-02, -1.7395e-02,  2.6230e-02,  1.4984e-02,\n",
       "                      -7.0740e-02,  1.3916e-02,  4.0512e-03, -1.8784e-02,  2.7725e-02,\n",
       "                       1.0078e-02,  4.7485e-02, -4.0855e-03, -3.6530e-02, -3.6011e-02,\n",
       "                       2.2079e-02, -3.2227e-02,  5.2166e-04, -3.1982e-02, -2.0493e-02,\n",
       "                       5.5237e-02,  2.4551e-02,  1.4374e-02, -5.9738e-03, -7.6111e-02,\n",
       "                       1.2962e-02,  3.2444e-03, -2.3804e-02, -1.8511e-03,  5.1346e-03,\n",
       "                      -5.8937e-03, -1.0353e-02, -3.4943e-02,  1.9855e-03, -1.4534e-02,\n",
       "                      -1.1421e-02,  7.4921e-03, -9.6069e-02,  1.2680e-02,  4.0009e-02,\n",
       "                      -1.8494e-02, -3.2711e-03,  2.1326e-01,  2.1759e-02, -5.2528e-03,\n",
       "                      -1.6098e-02, -6.7383e-02, -2.0142e-02,  7.0000e-03,  4.6356e-02,\n",
       "                       1.5579e-02, -2.7512e-02,  3.6346e-02, -2.7866e-03,  1.6312e-02,\n",
       "                       1.5007e-02,  2.9892e-02, -9.1171e-03,  2.5452e-02, -9.0332e-03,\n",
       "                       8.3374e-02, -4.7791e-02, -2.4796e-02])),\n",
       "             ('transformer.h.13.self_attention.dense.weight',\n",
       "              tensor([[ 0.0237, -0.0091,  0.0049,  ..., -0.0128,  0.0122,  0.0179],\n",
       "                      [-0.0086, -0.0210,  0.0468,  ..., -0.0595,  0.0100,  0.0417],\n",
       "                      [-0.0234,  0.0117, -0.0029,  ...,  0.0009, -0.0011, -0.0113],\n",
       "                      ...,\n",
       "                      [ 0.0188, -0.0278, -0.0080,  ...,  0.0058, -0.0466, -0.0056],\n",
       "                      [-0.0154,  0.0230, -0.0435,  ...,  0.0065,  0.0129, -0.0511],\n",
       "                      [ 0.0001, -0.0217, -0.0109,  ..., -0.0058, -0.0092, -0.0105]])),\n",
       "             ('transformer.h.13.self_attention.dense.bias',\n",
       "              tensor([ 0.0007,  0.0124,  0.0302,  ..., -0.0286, -0.0724,  0.0200])),\n",
       "             ('transformer.h.13.post_attention_layernorm.weight',\n",
       "              tensor([1.2061, 1.3711, 1.1367,  ..., 1.2900, 1.5166, 0.9087])),\n",
       "             ('transformer.h.13.post_attention_layernorm.bias',\n",
       "              tensor([ 0.0270, -0.0381,  0.0758,  ...,  0.1014,  0.0679,  0.1260])),\n",
       "             ('transformer.h.13.mlp.dense_h_to_4h.weight',\n",
       "              tensor([[-3.5187e-02, -3.6987e-02,  2.6112e-03,  ...,  2.9068e-02,\n",
       "                       -1.4664e-02, -1.9089e-02],\n",
       "                      [-4.9248e-03, -2.8000e-02,  7.5340e-03,  ..., -1.0132e-02,\n",
       "                       -8.2474e-03,  1.3268e-02],\n",
       "                      [-2.1851e-02, -1.4206e-02, -5.9906e-02,  ..., -7.6141e-03,\n",
       "                        9.6970e-03,  3.0594e-03],\n",
       "                      ...,\n",
       "                      [ 2.6642e-02,  2.9984e-02, -1.5869e-03,  ...,  3.0327e-03,\n",
       "                        1.9093e-03, -1.0643e-02],\n",
       "                      [ 1.7685e-02, -2.6276e-02,  5.0724e-05,  ...,  6.2904e-03,\n",
       "                        2.0782e-02, -2.1820e-02],\n",
       "                      [-1.6909e-03,  1.1238e-02,  3.8910e-03,  ..., -8.0185e-03,\n",
       "                        1.8723e-02, -3.4180e-02]])),\n",
       "             ('transformer.h.13.mlp.dense_h_to_4h.bias',\n",
       "              tensor([-0.0768, -0.0646,  0.0056,  ..., -0.4016, -0.4446, -0.0124])),\n",
       "             ('transformer.h.13.mlp.dense_4h_to_h.weight',\n",
       "              tensor([[ 0.0314, -0.0147,  0.0045,  ...,  0.0137, -0.0026,  0.0051],\n",
       "                      [ 0.0428,  0.0089,  0.0196,  ..., -0.0026,  0.0021, -0.0070],\n",
       "                      [-0.0107,  0.0129,  0.0544,  ...,  0.0249, -0.0123,  0.0076],\n",
       "                      ...,\n",
       "                      [-0.0165, -0.0114, -0.0117,  ...,  0.0091,  0.0011,  0.0065],\n",
       "                      [ 0.0075,  0.0177, -0.0154,  ...,  0.0018,  0.0353, -0.0172],\n",
       "                      [-0.0034, -0.0200,  0.0048,  ...,  0.0063, -0.0014, -0.0017]])),\n",
       "             ('transformer.h.13.mlp.dense_4h_to_h.bias',\n",
       "              tensor([ 0.0505, -0.1003, -0.1322,  ..., -0.1667,  0.2360, -0.0203])),\n",
       "             ('transformer.h.14.input_layernorm.weight',\n",
       "              tensor([1.3271, 1.7354, 1.2314,  ..., 1.8594, 1.8154, 0.9419])),\n",
       "             ('transformer.h.14.input_layernorm.bias',\n",
       "              tensor([-0.0035,  0.0595,  0.0414,  ...,  0.1593,  0.0516, -0.0263])),\n",
       "             ('transformer.h.14.self_attention.query_key_value.weight',\n",
       "              tensor([[ 0.0151,  0.0336,  0.0096,  ..., -0.0341,  0.0394,  0.0593],\n",
       "                      [-0.0034,  0.0550, -0.0124,  ...,  0.0195,  0.0168,  0.0112],\n",
       "                      [ 0.0154,  0.0041,  0.0135,  ..., -0.0093, -0.0182,  0.0210],\n",
       "                      ...,\n",
       "                      [-0.0192,  0.0247,  0.0419,  ..., -0.0019,  0.0167,  0.0170],\n",
       "                      [ 0.0124, -0.0175,  0.0247,  ...,  0.0032,  0.0194,  0.0183],\n",
       "                      [ 0.0270, -0.0049, -0.0027,  ...,  0.0107,  0.0158,  0.0529]])),\n",
       "             ('transformer.h.14.self_attention.query_key_value.bias',\n",
       "              tensor([-1.1853e-01, -5.0110e-02,  7.5073e-02,  6.6833e-02,  3.2593e-01,\n",
       "                       1.4343e-01, -8.4778e-02,  1.4587e-02, -1.2488e-01, -1.0358e-01,\n",
       "                      -1.3184e-01, -1.4978e-01,  7.9575e-03,  1.4880e-01, -2.6001e-01,\n",
       "                       7.4646e-02, -1.3049e-01, -2.9510e-02, -1.8311e-01, -3.9551e-02,\n",
       "                       1.3245e-01,  1.4502e-01,  8.9905e-02, -3.1113e-02,  1.1511e-01,\n",
       "                       1.4282e-01,  5.0720e-02,  1.8568e-03, -2.7319e-01,  2.1301e-01,\n",
       "                      -3.8727e-02, -3.6987e-02, -5.6091e-02, -7.6294e-02, -3.5034e-02,\n",
       "                       6.2408e-02, -2.6855e-03, -3.9612e-02, -8.6853e-02, -2.9640e-03,\n",
       "                       9.9915e-02, -7.8918e-02, -7.6599e-02,  1.0919e-01,  6.1981e-02,\n",
       "                      -1.3879e-01,  2.8854e-02,  4.4830e-02,  3.4393e-02, -3.6279e-01,\n",
       "                      -5.5359e-02,  4.4403e-02,  5.3284e-02, -4.3066e-01,  5.7098e-02,\n",
       "                      -3.8185e-03,  1.4648e+00,  7.7271e-02, -1.2708e-01, -1.7059e-02,\n",
       "                      -1.5210e-01,  1.4206e-02, -3.2379e-02,  1.2201e-01, -1.2413e-02,\n",
       "                      -2.3956e-02, -3.3691e-02,  7.8491e-02, -2.3285e-02, -3.1494e-02,\n",
       "                      -6.5689e-03,  2.1500e-02, -5.9166e-03,  5.7190e-02, -4.8113e-04,\n",
       "                      -7.1289e-02, -1.9150e-02,  3.9917e-02, -2.9236e-02,  2.4689e-02,\n",
       "                      -1.3496e-02, -4.1504e-02, -1.4709e-02, -4.4250e-02, -4.8706e-02,\n",
       "                      -4.9780e-01,  1.3298e-02,  1.4397e-02, -1.9836e-02, -2.7634e-02,\n",
       "                      -5.8655e-02,  5.2216e-02,  7.7972e-03, -2.1378e-02,  1.6510e-02,\n",
       "                      -7.6233e-02, -7.5317e-02, -3.3302e-03, -6.7566e-02, -6.9763e-02,\n",
       "                       9.5901e-03,  1.7044e-02, -6.7444e-03,  1.4244e-02, -3.7872e-02,\n",
       "                      -6.9763e-02, -3.7903e-02, -4.4281e-02, -7.5500e-02, -2.5162e-02,\n",
       "                       2.8503e-02, -1.9989e-02,  9.5825e-03, -2.8290e-02,  4.6204e-02,\n",
       "                       1.0818e-02,  3.9703e-02,  1.0635e-02, -9.3765e-03, -2.7390e-02,\n",
       "                       1.3268e-02,  1.3361e-03,  4.4189e-02, -1.2875e-03, -7.7400e-03,\n",
       "                       8.1253e-03,  7.0343e-03,  3.0003e-03,  4.5093e-01, -1.9324e-01,\n",
       "                       1.5752e+00,  2.6489e-01, -2.1082e-01,  1.4270e-01, -3.1763e-01,\n",
       "                      -2.3462e-01,  6.3171e-02, -9.3811e-02, -1.2048e-01, -2.5391e-02,\n",
       "                      -3.8745e-01,  1.8250e-01,  1.0065e-01, -3.7781e-02,  6.8604e-02,\n",
       "                      -1.1694e-01, -5.1709e-01,  5.7709e-02, -1.5854e-02, -9.9304e-02,\n",
       "                      -2.2498e-01, -3.2520e-01,  9.2102e-02, -2.7039e-02, -2.6443e-02,\n",
       "                      -3.6084e-01, -2.7612e-01,  5.5127e-01, -1.1212e-01, -1.6711e-01,\n",
       "                       2.0044e-01, -2.3712e-02, -9.0393e-02, -4.4189e-01,  1.5479e-01,\n",
       "                      -3.2422e-01, -7.1436e-01, -3.3545e-01,  1.1749e-01, -2.4512e+00,\n",
       "                       9.0137e-01,  3.0688e-01, -2.1069e-01,  2.9102e-01,  1.2671e-01,\n",
       "                       6.1963e-01,  3.0054e-01, -3.5718e-01, -1.3354e-01,  2.9834e-01,\n",
       "                      -5.4443e-02, -5.8929e-02,  2.8735e-01, -3.5187e-02,  9.1187e-02,\n",
       "                      -1.4502e-01,  2.0728e-01, -2.3401e-01, -5.7080e-01, -2.6440e-01,\n",
       "                      -1.3708e-01, -1.2219e-01, -8.4106e-02,  8.9788e-04,  6.9733e-03,\n",
       "                       5.4657e-02, -9.7107e-02, -6.0511e-04, -5.7251e-02,  1.2894e-03,\n",
       "                      -4.2633e-02,  4.0845e-01, -2.4292e-02, -2.2064e-02, -1.6199e-01,\n",
       "                       5.1025e-02,  4.2992e-03, -3.9581e-02,  1.0368e-02,  1.5305e-02,\n",
       "                      -1.8631e-02, -4.2328e-02,  5.6610e-02,  7.5874e-03, -3.7994e-02,\n",
       "                      -8.8196e-03, -3.4698e-02,  4.8889e-02, -1.4880e-01, -3.7140e-02,\n",
       "                       2.3804e-02,  4.1351e-02,  2.7344e-02, -5.9723e-02, -4.5441e-02,\n",
       "                      -2.8244e-02, -1.1475e-02, -5.2521e-02, -1.8509e-02, -5.8655e-02,\n",
       "                       3.8452e-02,  8.6609e-02, -3.2440e-02, -1.9974e-02,  4.2236e-02,\n",
       "                      -7.4539e-03,  2.8580e-02, -3.0174e-03,  7.3914e-02, -1.3199e-02,\n",
       "                      -6.9153e-02, -3.4882e-02, -5.5786e-02, -8.8120e-03, -1.4818e-04,\n",
       "                       9.0759e-02, -2.1851e-02, -2.0401e-02, -1.4923e-02, -4.6158e-03,\n",
       "                      -9.2010e-03,  7.6485e-03,  2.6489e-02,  2.2339e-02,  3.8605e-02,\n",
       "                      -2.3926e-02, -2.8931e-01,  6.6406e-02,  1.2817e-01, -1.0516e-01,\n",
       "                      -1.0419e-01, -1.1841e-01,  1.4197e-01, -1.8193e+00,  2.8488e-02,\n",
       "                       1.8555e-01, -1.7419e-01,  6.0693e-01, -2.5293e-01, -2.4902e-01,\n",
       "                      -1.7358e-01,  5.1904e-01,  1.1060e-01,  1.3721e-01,  2.2900e-01,\n",
       "                      -4.0112e-01,  7.9163e-02, -9.8648e-03,  1.8835e-01,  6.0352e-01,\n",
       "                       2.4976e-01,  5.7812e-01, -1.4961e+00,  2.2003e-02,  3.4546e-01,\n",
       "                      -5.9662e-02,  1.6687e-01,  2.4567e-03,  1.1096e-01,  4.0161e-01,\n",
       "                      -3.3838e-01,  1.8005e-01,  9.9487e-02,  3.9490e-02, -1.5601e-01,\n",
       "                      -3.4741e-01,  8.2703e-02,  3.0200e-01, -2.0645e+00,  5.8899e-02,\n",
       "                      -2.4866e-01,  2.5415e-01,  1.6309e-01, -1.3342e-01, -3.3984e-01,\n",
       "                       2.1912e-01,  2.1362e-01,  6.8994e-01, -3.7964e-02,  4.8169e-01,\n",
       "                       2.1594e-01,  3.5553e-02, -2.9434e-02,  2.9004e-01,  2.3987e-02,\n",
       "                       7.5562e-02,  9.0759e-02, -1.1945e-01,  1.7810e-01, -1.0931e-01,\n",
       "                      -3.9062e-02,  5.8105e-02,  4.7333e-02, -3.6255e-02,  4.3701e-02,\n",
       "                       3.2135e-02, -1.1795e-02,  4.4373e-02,  2.6970e-03,  4.8340e-02,\n",
       "                      -2.8229e-02,  2.9465e-02, -3.9558e-03,  2.4200e-02, -6.2134e-02,\n",
       "                       2.9877e-02, -2.3453e-02, -9.0576e-02,  1.7807e-02,  8.5068e-03,\n",
       "                       5.2917e-02,  2.7832e-02, -3.4981e-03,  8.2855e-03,  5.8643e-01,\n",
       "                       5.4718e-02,  8.2825e-02, -3.6035e-01,  3.8795e-03, -1.7509e-03,\n",
       "                      -3.2196e-02,  5.3711e-02, -5.0018e-02, -1.5732e-02, -1.0535e-01,\n",
       "                      -7.9590e-02, -7.9834e-02, -2.3174e-03,  6.4392e-02, -6.9237e-03,\n",
       "                       1.4526e-01, -2.8214e-02, -3.1677e-02, -3.2387e-03, -2.4078e-02,\n",
       "                      -4.7607e-03, -8.3923e-03,  4.1992e-02,  7.8186e-02,  6.8909e-02,\n",
       "                      -2.1912e-02,  1.5915e-02,  3.1403e-02, -1.0132e-02, -2.0370e-02,\n",
       "                       4.5898e-02,  2.1881e-02, -6.8542e-02, -5.9875e-02,  1.8967e-02,\n",
       "                       4.8485e-03, -1.2207e-02,  2.5558e-02, -1.2123e-02, -4.3365e-02,\n",
       "                       2.6318e-01, -2.4744e-01,  1.4832e-01, -2.1472e-01, -2.4570e+00,\n",
       "                       2.2888e-01,  4.6997e-02,  3.5205e-01, -3.5205e-01, -4.5074e-02,\n",
       "                      -2.3376e-01,  4.6851e-01,  4.2944e-01,  1.9360e-01,  1.5000e+00,\n",
       "                      -2.8442e-01, -2.4683e-01, -5.0879e-01,  2.8394e-01,  2.5342e-01,\n",
       "                       1.6309e-01,  1.9763e-01,  4.0063e-01, -1.9165e-01, -5.2521e-02,\n",
       "                       3.1079e-01, -2.7661e-01, -1.3354e-01,  2.7441e-01,  4.4128e-02,\n",
       "                      -9.9365e-02,  3.4839e-01, -2.9541e-01,  4.1406e-01, -2.3730e-01,\n",
       "                       1.5182e-02, -2.6562e-01,  2.7783e-01, -5.3271e-01, -2.0447e-01,\n",
       "                      -2.6367e-02,  8.2336e-02,  2.1790e-01,  1.6785e-01, -3.6335e-03,\n",
       "                       1.6357e-01, -9.4360e-02,  4.0601e-01,  7.8552e-02,  1.2830e-01,\n",
       "                      -1.4136e-01,  1.1005e-01,  2.0984e-01, -3.1543e-01, -5.1562e-01,\n",
       "                       2.8003e-01,  4.4775e-01, -3.3179e-01,  1.6479e-01, -1.6418e-02,\n",
       "                       2.7417e-01,  8.1787e-02,  4.7168e-01, -5.8252e-01, -3.6328e-01,\n",
       "                      -3.2495e-01,  2.8027e-01,  1.9714e-01,  4.9731e-01,  3.6755e-03,\n",
       "                       4.6997e-01, -4.6973e-01, -4.6094e-01,  3.4351e-01, -1.0669e-01,\n",
       "                      -3.7622e-01,  6.2500e-02,  1.7871e+00,  2.6428e-02, -3.1079e-01,\n",
       "                      -9.7839e-02, -1.8884e-01,  3.5205e-01,  4.8730e-01,  2.0898e-01,\n",
       "                       5.8594e-01,  6.2305e-01,  1.7932e-01, -2.3486e-01, -9.4629e-01,\n",
       "                       6.1737e-02, -1.8447e+00,  2.7173e-01, -2.7734e-01, -1.5198e-01,\n",
       "                       2.2815e-01,  1.5894e-01, -2.1558e-01,  3.7402e-01, -5.4541e-01,\n",
       "                       9.8926e-01,  7.4883e-03,  5.9509e-02,  3.2690e-01,  1.3599e-01,\n",
       "                      -2.6172e-01, -4.1504e-01,  7.6477e-02,  4.2920e-01, -1.0663e-01,\n",
       "                      -4.2871e-01, -1.6382e-01, -4.4214e-01, -4.1199e-02, -2.8320e-01,\n",
       "                      -5.5957e-01, -3.0566e-01,  3.9844e-01, -1.9739e-01,  1.0176e+00,\n",
       "                       2.3181e-01, -2.4121e-01,  1.2471e+00, -3.7402e-01,  7.1411e-02,\n",
       "                      -2.0117e-01, -2.4634e-01, -1.9543e-01, -1.6205e-02, -1.7627e-01,\n",
       "                       1.7517e-01,  9.0576e-02, -2.8638e-01,  8.7952e-02, -2.2485e-01,\n",
       "                       3.2300e-01,  2.2736e-02, -1.5976e-02,  3.0811e-01,  2.4048e-01,\n",
       "                      -1.2018e-01,  2.0203e-01,  2.3792e-01, -8.1421e-02,  8.6731e-02,\n",
       "                      -4.1553e-01, -3.5693e-01, -1.6370e-01,  6.6260e-01,  7.5623e-02,\n",
       "                       1.2476e-01, -1.9727e-01, -2.2107e-01, -5.4565e-02,  1.8311e-01,\n",
       "                      -2.6562e-01,  2.5562e-01, -2.8467e-01,  2.8027e-01, -2.2485e-01,\n",
       "                       3.5962e-01, -7.5256e-02,  7.3425e-02, -4.6906e-02,  1.0095e-01,\n",
       "                       3.7231e-03,  2.2034e-01,  3.4180e-02, -2.1704e-01,  5.7068e-02,\n",
       "                       1.5479e-01,  9.4788e-02,  9.2041e-01,  2.0227e-01,  5.0049e-01,\n",
       "                      -8.0139e-02, -1.3635e-01,  1.9995e-01, -8.5831e-03, -3.2056e-01,\n",
       "                      -1.1734e-02, -2.0386e-01,  2.3328e-01, -2.0840e+00,  1.7188e-01,\n",
       "                       3.2422e-01,  3.3008e-01,  1.5674e+00,  1.1591e-01, -4.4769e-02,\n",
       "                      -1.5540e-01, -1.4102e+00, -6.1523e-01, -1.0049e+00,  9.4043e-01,\n",
       "                      -1.1211e+00, -2.0203e-01,  1.3252e+00, -9.9316e-01,  9.9805e-01,\n",
       "                       7.2461e-01,  8.0713e-01,  3.2397e-01,  6.0059e-01, -1.5283e+00,\n",
       "                      -8.7939e-01,  1.5234e+00,  2.2144e-01,  9.0881e-02, -1.1562e+00,\n",
       "                      -2.4980e+00, -6.8115e-01,  7.9492e-01, -4.4238e-01, -2.3523e-01,\n",
       "                       4.6704e-01, -7.9541e-01, -1.6035e+00,  3.5962e-01,  9.6826e-01,\n",
       "                       1.7412e+00, -9.3359e-01,  2.0605e+00, -2.0195e+00,  1.8740e+00,\n",
       "                      -1.9153e-01,  8.5791e-01, -9.9731e-02,  7.9004e-01, -4.4019e-01,\n",
       "                       4.4336e-01,  2.0117e+00, -3.8721e-01,  1.1631e+00,  2.3975e-01,\n",
       "                       5.4736e-01,  3.3906e+00,  1.3369e+00,  4.2017e-01, -2.7344e-01,\n",
       "                      -1.1445e+00, -1.1127e-01, -1.1200e-01, -3.5095e-02, -8.7354e-01,\n",
       "                      -1.5566e+00,  1.2671e-01, -8.0391e+00,  1.3916e+00, -5.1758e-01,\n",
       "                       1.1279e+00,  6.1953e+00, -3.8306e-01, -4.4287e-01, -1.7402e+00,\n",
       "                       2.1899e-01, -2.6343e-01, -3.5083e-01, -4.0332e-01,  1.5051e-01,\n",
       "                       3.9917e-01, -3.0469e-01, -3.6279e-01, -2.2974e-01, -2.1741e-01,\n",
       "                      -1.1719e-02, -1.7651e-01,  5.2539e-01, -3.0350e-02,  2.1960e-01,\n",
       "                       1.6528e-01,  8.5022e-02,  4.2261e-01, -2.1497e-01,  4.6875e-02,\n",
       "                      -1.3782e-01, -6.3782e-02, -4.2896e-01,  2.3633e-01, -5.1074e-01,\n",
       "                      -1.7395e-01,  3.9917e-02, -2.2729e-01,  4.8340e-01, -4.3732e-02,\n",
       "                       6.4551e-01,  2.9617e-02,  3.9624e-01, -1.9971e+00,  1.9128e-01,\n",
       "                      -3.4595e-01,  3.2593e-01,  5.7959e-01, -1.7163e-01, -2.0538e-02,\n",
       "                      -1.7603e-01,  5.8887e-01, -1.9141e-01, -1.4441e-01, -3.6957e-02,\n",
       "                       3.4473e-01, -1.0168e-01,  3.0518e-01, -7.2656e-01,  3.2257e-02,\n",
       "                      -2.2998e-01, -1.9473e+00,  1.9463e+00, -1.3464e-01,  4.1772e-01,\n",
       "                       1.3538e-01,  2.8564e-01, -3.6133e-02,  6.2195e-02, -2.8223e-01,\n",
       "                       1.9189e+00, -4.0332e-01, -7.2705e-01,  2.8662e-01, -4.2798e-01,\n",
       "                      -2.0117e+00,  1.2201e-01,  2.0391e+00,  8.1055e-01,  8.2178e-01,\n",
       "                      -5.9717e-01, -5.0098e-01,  3.1909e-01,  7.3398e+00, -9.6619e-02,\n",
       "                       2.7520e+00,  9.1094e+00,  7.1094e-01,  1.0010e+00,  3.9478e-01,\n",
       "                       7.7246e-01, -2.1016e+00,  2.2520e+00,  3.2837e-01,  8.9551e-01,\n",
       "                       4.0918e-01, -1.5576e-01,  9.9316e-01,  4.0576e-01, -6.3574e-01,\n",
       "                       9.0381e-01, -2.0312e+00,  2.5234e+00,  5.6055e-01,  9.8047e-01,\n",
       "                       1.7148e+00,  3.0737e-01,  8.0908e-01, -9.3115e-01,  5.6104e-01,\n",
       "                       9.3359e-01,  7.0508e-01, -1.0078e+00,  8.3398e-01, -4.0314e-02,\n",
       "                       5.1611e-01, -1.2500e-01, -2.9570e+00,  1.8445e-01, -4.5801e-01,\n",
       "                      -2.3379e+00,  1.5010e+00, -1.2772e-02,  3.2837e-01, -2.2070e-01,\n",
       "                      -3.8232e-01, -7.1953e+00, -7.0264e-01, -5.8359e+00, -1.9824e+00,\n",
       "                      -5.5176e-01, -1.6006e+00,  3.8086e-01, -9.6436e-01, -6.6455e-01,\n",
       "                      -4.1875e+00,  3.8037e-01, -8.0762e-01,  1.7914e-02,  6.8512e-03,\n",
       "                       1.7639e-02, -2.2232e-02, -5.7007e-02, -1.2642e-02,  7.3975e-02,\n",
       "                       4.0283e-03,  2.8061e-02,  1.2390e-02, -3.0319e-02,  2.2476e-02,\n",
       "                       3.0308e-03, -6.2065e-03, -5.6702e-02, -9.4986e-03,  4.4250e-02,\n",
       "                       6.5491e-02, -6.5918e-02, -2.8259e-02,  6.5308e-02,  1.2383e-02,\n",
       "                      -2.1286e-02, -1.6830e-02, -8.6670e-03, -8.9645e-03, -3.1235e-02,\n",
       "                       4.2725e-04, -3.4637e-02, -1.4008e-02,  4.1389e-03,  8.6136e-03,\n",
       "                      -1.2611e-02, -2.7313e-02, -7.9956e-02, -2.3453e-02, -1.5518e-02,\n",
       "                       7.4768e-03,  5.0598e-02, -1.3329e-02, -7.0801e-02, -5.5603e-02,\n",
       "                       1.3680e-02, -3.6285e-02, -9.9258e-03,  2.2263e-02, -1.4931e-02,\n",
       "                       2.0782e-02, -3.2074e-02, -1.0963e-02,  1.8509e-02,  4.8904e-03,\n",
       "                       1.9318e-02,  7.6103e-03, -1.1505e-02,  2.6428e-02, -1.7303e-02,\n",
       "                      -1.8005e-02, -4.6234e-02, -3.9001e-02, -1.2764e-02,  5.3549e-04,\n",
       "                      -3.2654e-03, -4.1473e-02,  6.6455e-01, -1.4111e-01, -1.5957e+00,\n",
       "                       4.2065e-01, -1.3779e+00, -1.1631e+00,  1.1826e+00, -4.5410e-01,\n",
       "                      -1.6980e-01,  1.0059e+00,  1.5635e+00, -1.6084e+00, -7.6367e-01,\n",
       "                      -8.6719e-01,  8.4717e-01, -1.0908e+00, -1.7871e+00, -7.2656e-01,\n",
       "                       9.7266e-01,  2.1934e+00,  6.8164e-01, -6.2561e-02,  3.1289e+00,\n",
       "                      -1.2494e-01, -9.7852e-01,  1.5420e+00,  1.0488e+00,  1.2510e+00,\n",
       "                       3.1494e-01, -2.8438e+00,  1.2617e+00,  9.5215e-01,  7.3535e-01,\n",
       "                      -7.6855e-01,  1.7324e+00,  1.2773e+00, -1.9814e+00,  2.2614e-02,\n",
       "                       6.8018e-01, -1.1836e+00,  9.1309e-02,  5.0781e-01,  3.7231e-01,\n",
       "                      -4.5459e-01,  4.7461e-01,  3.6230e-01, -4.5868e-02, -1.0967e+00,\n",
       "                       1.9590e+00, -5.1910e-02, -6.1426e-01, -3.4380e-04, -7.1680e-01,\n",
       "                       5.9277e-01, -2.3242e+00, -7.3047e-01, -4.9902e-01,  9.2725e-01,\n",
       "                       2.5332e+00, -8.7744e-01, -1.5391e+00,  1.1826e+00,  2.9526e-02,\n",
       "                      -1.5566e+00,  2.7740e-02, -3.0151e-02, -2.5574e-02, -4.0169e-03,\n",
       "                       2.3098e-03,  1.8890e-02,  5.5481e-02,  1.3741e-02,  1.9211e-02,\n",
       "                       6.4774e-03, -9.6416e-04,  1.0811e-02, -1.1551e-02,  4.1931e-02,\n",
       "                      -2.5421e-02,  6.0883e-02,  5.2155e-02,  2.8992e-03,  6.8474e-03,\n",
       "                       7.9651e-03,  1.6968e-02,  1.0017e-02,  3.0502e-02,  1.1772e-02,\n",
       "                       6.1798e-03, -5.4436e-03, -3.1686e-04,  5.7173e-04,  1.7105e-02,\n",
       "                      -1.7593e-02, -1.7639e-02, -1.2009e-02, -1.3649e-02,  2.5543e-02,\n",
       "                      -2.5177e-02,  2.0370e-02, -4.6883e-03, -9.4223e-03, -3.6499e-02,\n",
       "                       2.9770e-02, -1.8826e-03, -1.0597e-02, -2.1072e-02, -3.1525e-02,\n",
       "                      -3.2616e-04,  4.5197e-02,  2.1469e-02, -3.6297e-03,  4.4342e-02,\n",
       "                      -3.1113e-02, -1.3885e-02,  2.6184e-02, -7.3318e-03,  2.8946e-02,\n",
       "                       9.6703e-04,  8.4305e-03, -8.4991e-03, -5.0079e-02, -4.1412e-02,\n",
       "                      -3.2673e-03, -3.2623e-02,  3.2692e-03, -3.8586e-03, -2.0538e-02])),\n",
       "             ('transformer.h.14.self_attention.dense.weight',\n",
       "              tensor([[ 0.0165, -0.0073,  0.0075,  ...,  0.0272,  0.0161, -0.0072],\n",
       "                      [ 0.0196, -0.0052,  0.0042,  ...,  0.0100,  0.0004, -0.0169],\n",
       "                      [ 0.0051,  0.0139,  0.0210,  ...,  0.0069,  0.0046,  0.0137],\n",
       "                      ...,\n",
       "                      [ 0.0198, -0.0167,  0.0165,  ...,  0.0421, -0.0128,  0.0002],\n",
       "                      [ 0.0049, -0.0104,  0.0462,  ...,  0.0015,  0.0175, -0.0098],\n",
       "                      [-0.0116, -0.0074, -0.0089,  ..., -0.0215,  0.0131, -0.0183]])),\n",
       "             ('transformer.h.14.self_attention.dense.bias',\n",
       "              tensor([ 0.0089,  0.0197,  0.0431,  ...,  0.0038, -0.0807,  0.0145])),\n",
       "             ('transformer.h.14.post_attention_layernorm.weight',\n",
       "              tensor([1.2529, 1.3525, 1.1963,  ..., 1.2686, 1.5264, 0.9629])),\n",
       "             ('transformer.h.14.post_attention_layernorm.bias',\n",
       "              tensor([0.0526, 0.0228, 0.0110,  ..., 0.0699, 0.0834, 0.1442])),\n",
       "             ('transformer.h.14.mlp.dense_h_to_4h.weight',\n",
       "              tensor([[ 0.0122, -0.0338, -0.0217,  ...,  0.0338, -0.0302,  0.0057],\n",
       "                      [-0.0457, -0.0049, -0.0002,  ..., -0.0027,  0.0185,  0.0044],\n",
       "                      [-0.0011,  0.0033,  0.0152,  ...,  0.0295,  0.0016, -0.0200],\n",
       "                      ...,\n",
       "                      [-0.0207,  0.0173,  0.0054,  ..., -0.0008,  0.0008, -0.0108],\n",
       "                      [-0.0095,  0.0135,  0.0282,  ...,  0.0054, -0.0093,  0.0004],\n",
       "                      [ 0.0065,  0.0093, -0.0176,  ..., -0.0190, -0.0247,  0.0066]])),\n",
       "             ('transformer.h.14.mlp.dense_h_to_4h.bias',\n",
       "              tensor([-0.4802, -0.3589, -0.3755,  ..., -0.3555, -0.2568, -0.0916])),\n",
       "             ('transformer.h.14.mlp.dense_4h_to_h.weight',\n",
       "              tensor([[-0.0043, -0.0109,  0.0055,  ..., -0.0051,  0.0060, -0.0254],\n",
       "                      [-0.0170, -0.0209, -0.0020,  ..., -0.0155, -0.0228,  0.0046],\n",
       "                      [-0.0135,  0.0030,  0.0098,  ...,  0.0023, -0.0036, -0.0019],\n",
       "                      ...,\n",
       "                      [ 0.0139,  0.0202,  0.0219,  ...,  0.0171, -0.0263,  0.0080],\n",
       "                      [-0.0039,  0.0085,  0.0008,  ...,  0.0209,  0.0161,  0.0062],\n",
       "                      [ 0.0041,  0.0063, -0.0093,  ...,  0.0083, -0.0139,  0.0023]])),\n",
       "             ('transformer.h.14.mlp.dense_4h_to_h.bias',\n",
       "              tensor([ 0.0630, -0.1255, -0.1389,  ..., -0.2111,  0.2003, -0.0072])),\n",
       "             ('transformer.h.15.input_layernorm.weight',\n",
       "              tensor([1.4141, 1.7529, 1.3242,  ..., 1.7109, 1.7803, 1.0996])),\n",
       "             ('transformer.h.15.input_layernorm.bias',\n",
       "              tensor([ 0.0166,  0.0434,  0.0177,  ...,  0.2087,  0.0658, -0.0205])),\n",
       "             ('transformer.h.15.self_attention.query_key_value.weight',\n",
       "              tensor([[-1.2749e-02, -6.2485e-03,  3.3665e-03,  ..., -2.5902e-03,\n",
       "                        3.1158e-02,  2.5192e-02],\n",
       "                      [-9.3758e-05, -9.2087e-03, -7.2289e-03,  ...,  1.0849e-02,\n",
       "                        7.0457e-03,  2.2430e-02],\n",
       "                      [-3.2349e-02,  7.1259e-03,  3.7136e-03,  ..., -7.0229e-03,\n",
       "                       -1.1093e-02, -1.4221e-02],\n",
       "                      ...,\n",
       "                      [ 3.5278e-02,  7.2899e-03,  2.0630e-02,  ...,  1.7914e-02,\n",
       "                       -2.3788e-02,  2.8152e-02],\n",
       "                      [ 2.7054e-02, -1.1620e-02,  2.7023e-02,  ..., -3.6354e-03,\n",
       "                       -1.7365e-02, -2.4048e-02],\n",
       "                      [ 5.4565e-02, -1.4526e-02,  3.9429e-02,  ..., -3.4313e-03,\n",
       "                        3.3798e-03,  3.2654e-02]])),\n",
       "             ('transformer.h.15.self_attention.query_key_value.bias',\n",
       "              tensor([ 5.3864e-02, -1.0887e-02,  2.9358e-02, -2.6001e-02, -1.4809e-02,\n",
       "                       1.7563e-02,  3.3691e-02, -8.7891e-02, -3.2063e-03,  8.4106e-02,\n",
       "                       3.6812e-03, -2.2293e-02,  2.6047e-02, -1.4725e-02,  2.2995e-02,\n",
       "                       1.9241e-02, -3.4760e-02, -4.7638e-02,  1.0071e-02, -1.7273e-02,\n",
       "                       1.1493e-01,  1.3657e-03,  2.9999e-02,  8.4686e-03,  2.0279e-02,\n",
       "                      -1.1749e-03, -3.8147e-02,  3.9215e-02, -1.4977e-02, -4.4647e-02,\n",
       "                      -1.9627e-03,  1.1024e-02,  6.1310e-02, -1.3397e-02, -8.3557e-02,\n",
       "                       3.0136e-02,  1.8930e-04, -4.8180e-03, -3.2959e-02, -1.0468e-02,\n",
       "                       9.2697e-03, -1.5593e-04, -1.4481e-02, -5.3596e-04, -1.3206e-02,\n",
       "                      -5.4565e-02, -4.4617e-02,  3.2013e-02,  7.6332e-03, -5.1117e-03,\n",
       "                       2.9892e-02,  4.6661e-02,  6.3553e-03,  4.9622e-02, -4.8523e-02,\n",
       "                      -3.3081e-02, -1.5976e-02,  4.9782e-03, -1.6556e-02,  1.3992e-02,\n",
       "                       1.1452e-02, -2.5742e-02,  7.9956e-02,  2.0630e-02, -5.2393e-01,\n",
       "                       1.5320e-01, -2.4658e-01,  1.2115e-01, -3.4668e-01,  2.3340e-01,\n",
       "                      -1.1707e-01, -4.8901e-01,  3.7262e-02, -4.5239e-01, -4.7974e-02,\n",
       "                       3.9886e-02,  6.1981e-02,  1.5735e-01,  1.5710e-01,  1.6516e-01,\n",
       "                       2.5952e-01,  5.6543e-01, -1.6870e-01, -8.5754e-02, -2.8345e-01,\n",
       "                       1.2646e-01,  2.1602e+00, -3.3997e-02,  5.7178e-01, -5.6732e-02,\n",
       "                       2.6343e-01, -2.8418e-01,  1.2238e-01,  2.2693e-01, -2.4890e-01,\n",
       "                       6.4258e-01, -2.8784e-01,  8.5510e-02, -7.5439e-02,  8.5815e-02,\n",
       "                      -1.8237e-01, -4.7668e-02,  2.5977e-01, -2.8595e-02, -1.1890e-01,\n",
       "                       2.5620e-02, -9.0698e-02,  3.1909e-01,  5.4455e-04, -2.3340e-01,\n",
       "                      -1.2903e-01, -3.3960e-01, -1.4746e-01, -1.2268e-02, -9.2896e-02,\n",
       "                      -3.9380e-01,  7.3242e-02, -3.8391e-02,  5.6934e-01,  1.3525e-01,\n",
       "                       9.0637e-03, -3.3887e-01,  1.8525e+00,  9.3140e-02, -2.7441e-01,\n",
       "                       8.7952e-02, -9.4299e-02,  2.6562e-01, -1.3403e-01,  3.7842e-02,\n",
       "                      -7.3433e-03,  8.3557e-02, -1.6040e-01, -8.7769e-02, -8.2886e-02,\n",
       "                      -1.6650e-01,  1.2219e-01, -1.8982e-01,  1.0547e-01,  1.3342e-01,\n",
       "                      -2.3026e-02, -8.5693e-02,  3.9886e-02,  4.0192e-02,  1.2317e-01,\n",
       "                       1.1505e-01,  4.3702e-04,  6.7993e-02, -6.1859e-02,  2.4597e-02,\n",
       "                       1.6426e+00, -9.6985e-02,  1.1017e-01, -7.3059e-02, -1.5976e-02,\n",
       "                      -1.6418e-01,  3.6469e-02,  1.8286e-01, -1.0626e-01,  3.0322e-01,\n",
       "                      -1.1438e-01,  2.1545e-01,  1.3748e-02,  2.6062e-02,  1.9897e-02,\n",
       "                      -6.8970e-02,  1.0455e-01, -3.2776e-02, -3.6850e-03,  5.8365e-04,\n",
       "                       2.9022e-02,  2.0142e-01, -1.7303e-02, -4.7058e-02, -3.1311e-02,\n",
       "                      -1.3232e-01, -1.0193e-01, -8.2397e-02,  5.1605e-02, -9.9060e-02,\n",
       "                      -1.1401e-01, -6.0852e-02,  2.1655e-01,  3.9490e-02, -2.9465e-02,\n",
       "                      -8.7814e-03,  1.4287e+00,  9.7534e-02,  2.0477e-02,  2.8595e-02,\n",
       "                      -5.9242e-03,  1.4819e-01, -1.0345e-01, -1.5125e-01, -4.0991e-01,\n",
       "                      -3.7975e-03,  5.1514e-01,  9.0820e-01,  3.0420e-01, -1.3662e+00,\n",
       "                      -2.8198e-01,  1.5051e-01,  4.8682e-01, -4.4067e-02, -2.3633e-01,\n",
       "                      -7.7858e-03,  2.0032e-01,  1.9910e-01,  2.9102e-01,  7.6758e-01,\n",
       "                       3.5889e-01,  2.1741e-01, -1.5735e-01,  3.0701e-02, -2.9639e-01,\n",
       "                       2.4509e-03, -4.9121e-01, -2.3164e+00,  2.7588e-01,  5.4004e-01,\n",
       "                       1.5686e-01, -3.0493e-01, -2.3828e-01, -2.0483e-01, -3.5571e-01,\n",
       "                       9.1370e-02, -3.0258e-02,  3.4973e-02,  1.0858e-01, -2.8271e-01,\n",
       "                      -1.2524e-01,  5.3271e-01, -4.5020e-01, -1.7078e-01, -2.4219e-01,\n",
       "                      -1.9336e-01, -1.7871e-01,  2.9373e-02,  1.2408e-01, -1.9043e-01,\n",
       "                       1.5234e-01, -1.3501e-01,  7.3975e-02,  8.1848e-02, -5.2637e-01,\n",
       "                       5.0415e-02, -5.2539e-01,  4.8767e-02,  2.1992e+00, -4.8706e-01,\n",
       "                      -7.5317e-02,  7.2754e-02, -9.1260e-01, -3.8867e-01,  2.5366e-01,\n",
       "                       3.7866e-01,  5.0879e-01,  2.2424e-01, -3.3276e-01,  6.5234e-01,\n",
       "                      -3.4717e-01, -2.1118e-01,  3.9282e-01, -9.7107e-02, -3.4741e-01,\n",
       "                       2.2192e-01, -1.4282e-01,  8.0338e-03,  7.2412e-01, -4.7778e-01,\n",
       "                       3.9856e-02,  1.7285e-01,  4.0283e-01, -8.9844e-01,  2.6074e+00,\n",
       "                      -1.3291e+00, -6.9873e-01, -6.1963e-01, -4.9463e-01,  6.8169e-03,\n",
       "                      -2.7368e-01,  3.8062e-01, -5.2734e-01, -3.1714e-01, -7.0850e-01,\n",
       "                      -1.9849e-01,  5.3223e-01, -3.3539e-02, -4.0845e-01, -5.7227e-01,\n",
       "                      -5.7178e-01, -3.6084e-01, -5.1611e-01,  5.3369e-01,  9.2346e-02,\n",
       "                       1.0626e-01, -8.5303e-01, -5.0049e-01,  1.2512e-01,  1.9019e-01,\n",
       "                      -2.7686e-01, -1.9958e-01, -1.1865e-01,  2.6147e-01,  7.7051e-01,\n",
       "                       5.2881e-01, -5.4736e-01,  2.9248e-01, -3.6890e-01, -2.0471e-01,\n",
       "                      -3.7012e-01,  3.5913e-01, -5.3662e-01,  6.9482e-01,  8.6377e-01,\n",
       "                      -9.0271e-02,  2.2095e-01, -2.1399e-01, -4.7699e-02,  1.6284e-01,\n",
       "                       3.1445e+00,  1.7590e-01,  6.7322e-02,  5.7080e-01,  7.2188e+00,\n",
       "                       1.0547e+00, -2.3848e+00,  1.0977e+00, -6.9763e-02,  2.9614e-01,\n",
       "                       2.7266e+00,  5.2539e-01,  2.6562e+00, -2.4043e+00, -5.9473e-01,\n",
       "                       1.5605e+00, -2.8887e+00, -2.0859e+00,  1.6094e+00, -6.0205e-01,\n",
       "                      -2.0625e+00,  3.4727e+00,  1.3018e+00,  1.7566e-01,  1.5068e+00,\n",
       "                      -2.1367e+00, -7.6562e-01, -2.9824e+00,  1.4600e-01,  1.3047e+00,\n",
       "                      -2.9614e-01, -3.9316e+00, -6.3330e-01,  9.6582e-01,  2.1230e+00,\n",
       "                       2.5938e+00,  1.9409e-01,  1.6150e-01, -1.8760e+00, -2.1133e+00,\n",
       "                       2.2930e+00,  1.5244e+00,  5.9277e-01, -9.3701e-01, -1.1895e+00,\n",
       "                      -2.8931e-01,  1.1885e+00, -1.8203e+00, -4.8389e-01, -1.3564e+00,\n",
       "                      -6.0699e-02,  2.2637e+00, -2.3105e+00, -1.5430e+00,  2.5078e+00,\n",
       "                       3.6836e+00,  2.7520e+00, -2.1035e+00,  1.9946e-01, -1.8484e+01,\n",
       "                       1.6357e+00, -5.0195e-01,  2.1992e+00,  1.6523e+00,  3.4866e-03,\n",
       "                      -7.7591e-03, -4.4159e-02,  2.1118e-02,  2.5070e-02,  6.4545e-03,\n",
       "                       1.3893e-02,  3.4599e-03, -3.9276e-02,  1.1665e-02,  1.6510e-02,\n",
       "                       1.8433e-02, -5.5756e-02, -2.1378e-02, -1.7729e-03, -1.6632e-02,\n",
       "                       5.7030e-03, -1.6832e-03,  7.6370e-03,  3.1948e-03,  1.5160e-02,\n",
       "                       1.9180e-02,  1.1696e-02, -4.3732e-02,  1.7029e-02,  9.4833e-03,\n",
       "                      -5.0392e-03,  5.8594e-03,  1.1040e-02,  9.2239e-03, -1.4778e-02,\n",
       "                       1.0567e-02,  6.9733e-03,  4.4525e-02,  9.2010e-03, -5.3368e-03,\n",
       "                       2.3972e-02,  3.7880e-03, -4.0855e-03, -2.7023e-02,  4.3243e-02,\n",
       "                       2.9617e-02, -3.8624e-03,  5.4016e-03, -3.7861e-03,  1.2268e-02,\n",
       "                       3.5133e-03, -1.2733e-02, -1.0718e-01, -2.5696e-02,  1.9165e-02,\n",
       "                       8.6899e-03,  1.4954e-02, -1.6870e-03,  4.5471e-03,  1.0429e-02,\n",
       "                       1.4694e-02, -4.5410e-02, -4.9477e-03,  6.1455e-03, -2.0370e-03,\n",
       "                       6.3591e-03, -1.9226e-02,  3.9490e-02, -4.7344e+00, -1.9521e+00,\n",
       "                      -3.1113e+00, -1.6982e+00,  2.9082e+00,  6.7891e+00,  1.1387e+00,\n",
       "                       8.3359e+00, -1.4873e+00, -3.4004e+00,  5.0830e-01, -1.7471e+00,\n",
       "                      -2.6504e+00, -1.6541e-01, -6.7695e+00,  3.1406e+01,  3.7129e+00,\n",
       "                       2.5547e+00,  5.7227e+00, -2.1016e+00, -4.1914e+00,  1.1992e+00,\n",
       "                       4.7266e+00, -1.9951e+00, -3.9336e+00, -1.3418e+00, -2.0141e+01,\n",
       "                       3.3613e+00, -3.2852e+00,  3.3496e+00,  5.3047e+00, -6.7090e-01,\n",
       "                       4.3945e+00,  3.1484e+01,  4.4180e+00, -6.0107e-01,  2.2676e+00,\n",
       "                       6.7188e+00, -3.5703e+00, -2.8086e+00, -9.7891e+00,  2.9473e+00,\n",
       "                       5.6562e+00,  4.5430e+00,  6.3281e+00, -3.0430e+00,  6.1328e+00,\n",
       "                       2.7578e+00,  2.0410e+00,  4.3008e+00,  2.9590e+00,  1.8594e+00,\n",
       "                       5.1367e-01, -3.0000e+00,  2.0020e+00,  8.1982e-01,  2.7125e+01,\n",
       "                       5.7891e+00, -6.1836e+00, -1.7285e+00,  7.0752e-01, -2.0332e+00,\n",
       "                       8.1406e+00,  5.3281e+00,  9.1943e-01,  1.3574e+00, -1.6758e+00,\n",
       "                      -2.4219e+00,  1.9958e-01, -9.9658e-01,  7.9785e-01,  2.0293e+00,\n",
       "                       1.2988e+00,  1.2998e+00,  2.6113e+00,  1.1387e+00,  2.8359e+00,\n",
       "                       8.8440e-02, -1.3721e+00,  5.5127e-01,  1.4395e+00,  2.3027e+00,\n",
       "                       2.8301e+00,  2.8184e+00, -2.5156e+00,  2.0059e+00, -1.2646e+00,\n",
       "                       7.6270e-01, -9.6191e-01,  7.7734e-01, -4.6045e-01,  3.3398e+00,\n",
       "                       7.8711e-01,  5.5420e-01,  6.7383e-01,  1.2393e+00,  1.1475e+00,\n",
       "                       7.3047e-01, -1.5889e+00,  1.3779e+00,  1.6809e-01, -1.0713e+00,\n",
       "                       7.0752e-01, -8.4912e-01,  6.9189e-01, -3.3447e-01,  9.8340e-01,\n",
       "                      -5.1074e-01,  2.6718e-02, -1.9971e-01, -2.5293e+00, -4.2920e-01,\n",
       "                      -8.2666e-01, -1.1797e+00,  1.8418e+00, -1.3799e+00,  7.4121e-01,\n",
       "                       1.1338e+00, -1.7891e+00,  1.3945e+00,  1.1511e-01, -1.6582e+00,\n",
       "                      -4.3433e-01,  2.4048e-02, -1.0293e+00,  2.3779e-01, -3.2715e-02,\n",
       "                       2.1997e-01, -8.4167e-02, -2.4307e-02, -4.4037e-02, -3.7933e-02,\n",
       "                      -5.6686e-03, -2.7206e-02,  1.8097e-02, -3.7292e-02, -2.8641e-02,\n",
       "                       1.8173e-02,  1.6083e-02,  1.8631e-02,  2.5208e-02,  1.9745e-02,\n",
       "                       1.7258e-02, -7.6294e-03, -6.3660e-02,  4.0398e-03, -3.4576e-02,\n",
       "                       5.9143e-02, -1.4893e-02,  3.2532e-02,  7.3700e-03,  5.1971e-02,\n",
       "                       2.0523e-02,  8.8959e-03,  5.1331e-02,  2.6184e-02, -1.3657e-02,\n",
       "                      -4.5715e-02, -3.0060e-02,  9.2896e-02,  1.1330e-02,  1.8188e-02,\n",
       "                      -6.9031e-02,  4.2200e-04, -3.7842e-02,  2.0203e-02,  6.4270e-02,\n",
       "                      -1.6232e-03, -1.2947e-02, -4.2725e-02, -7.9590e-02,  4.9561e-02,\n",
       "                       1.0217e-01, -3.6346e-02,  1.0277e-02,  3.6499e-02, -8.7646e-02,\n",
       "                      -5.0018e-02, -1.1253e-02,  1.2188e-03, -5.3253e-02, -6.4880e-02,\n",
       "                       1.6861e-02,  2.9099e-02,  5.1155e-03, -2.6596e-02,  2.8038e-03,\n",
       "                      -6.8237e-02, -1.4587e-02,  1.3786e-02, -5.0720e-02, -2.5909e-02,\n",
       "                      -1.1513e-02,  9.0149e-02, -1.1639e-01, -1.4087e-01, -1.1864e-02,\n",
       "                      -7.5928e-02, -3.6938e-01, -1.2830e-01, -5.1544e-02,  6.7871e-02,\n",
       "                      -7.3730e-02, -8.6365e-02,  1.1255e-01,  5.4443e-02, -5.7220e-02,\n",
       "                      -2.0447e-01, -1.9852e-02,  6.2042e-02, -1.3838e+00,  1.1353e-01,\n",
       "                      -1.6541e-01,  2.7490e-01, -6.0028e-02, -7.0496e-02,  8.0933e-02,\n",
       "                      -6.1523e-02,  5.5695e-02,  5.9448e-02, -7.9163e-02,  1.4900e-02,\n",
       "                       1.7100e+00, -8.9050e-02, -5.2100e-01,  3.7048e-02,  7.8735e-02,\n",
       "                      -2.6343e-01,  2.7756e-02, -1.2527e-02, -1.1090e-01,  4.1748e-02,\n",
       "                      -4.3396e-02,  1.6492e-01,  1.4473e-02,  1.6602e-01, -1.2732e-01,\n",
       "                       9.3613e-03,  4.9835e-02,  1.4551e-01, -1.6203e-03,  4.4403e-02,\n",
       "                       1.9617e-01, -1.1261e-01,  2.0660e-02, -2.5195e-01,  1.4197e-01,\n",
       "                       1.4121e+00, -3.9307e-02,  4.6509e-02, -3.1067e-02,  5.3314e-02,\n",
       "                       1.0327e-01,  1.2927e-01, -2.2937e-01, -1.5479e-01, -1.8631e-02,\n",
       "                       5.7587e-02,  3.8033e-03, -2.0126e-02,  2.6264e-03, -1.5335e-02,\n",
       "                      -9.2163e-03,  2.0294e-03,  1.0674e-02, -4.1382e-02,  7.2403e-03,\n",
       "                       3.4363e-02,  3.5553e-02,  8.2932e-03,  5.7869e-03,  2.6230e-02,\n",
       "                       1.2268e-02, -2.5009e-02,  1.1772e-02,  5.8289e-03, -7.8125e-03,\n",
       "                      -4.7112e-03, -2.4323e-02,  2.3605e-02, -2.2352e-04, -3.1143e-02,\n",
       "                      -1.0895e-02, -3.7956e-03,  1.9806e-02, -4.9629e-03, -2.9373e-02,\n",
       "                      -1.0124e-02, -2.9755e-02, -4.7798e-03, -8.5297e-03, -2.3056e-02,\n",
       "                       4.4289e-03,  3.0624e-02, -5.3802e-02,  1.7578e-02, -7.0877e-03,\n",
       "                      -7.6294e-04,  2.3682e-02, -2.6215e-02,  2.5539e-03, -4.1084e-03,\n",
       "                      -3.5187e-02, -5.4436e-03,  9.4604e-03, -5.0201e-03, -1.2207e-02,\n",
       "                      -1.8799e-02, -2.8488e-02,  1.2413e-02,  2.0325e-02,  1.5945e-02,\n",
       "                      -2.6947e-02,  4.3144e-03, -2.0538e-02, -4.3304e-02,  1.8478e-02,\n",
       "                       4.5395e-03,  1.6724e-02,  3.1281e-02])),\n",
       "             ('transformer.h.15.self_attention.dense.weight',\n",
       "              tensor([[ 0.0203, -0.0092,  0.0293,  ..., -0.0285, -0.0376, -0.0427],\n",
       "                      [-0.0009,  0.0007, -0.0232,  ..., -0.0143, -0.0052,  0.0094],\n",
       "                      [-0.0209,  0.0177, -0.0275,  ..., -0.0014, -0.0361, -0.0627],\n",
       "                      ...,\n",
       "                      [-0.0032,  0.0005, -0.0014,  ..., -0.0092,  0.0033,  0.0213],\n",
       "                      [-0.0102, -0.0143,  0.0033,  ...,  0.0143,  0.0173,  0.0015],\n",
       "                      [-0.0396,  0.0032,  0.0291,  ..., -0.0389, -0.0093, -0.0269]])),\n",
       "             ('transformer.h.15.self_attention.dense.bias',\n",
       "              tensor([-0.0090,  0.0110,  0.0552,  ..., -0.0098, -0.0665,  0.0326])),\n",
       "             ('transformer.h.15.post_attention_layernorm.weight',\n",
       "              tensor([1.2861, 1.3945, 1.2168,  ..., 1.2080, 1.5000, 1.0098])),\n",
       "             ('transformer.h.15.post_attention_layernorm.bias',\n",
       "              tensor([ 0.0533, -0.0069,  0.0139,  ...,  0.1733,  0.0732,  0.1036])),\n",
       "             ('transformer.h.15.mlp.dense_h_to_4h.weight',\n",
       "              tensor([[ 0.0074, -0.0126, -0.0060,  ..., -0.0004, -0.0195, -0.0188],\n",
       "                      [-0.0067, -0.0319, -0.0088,  ...,  0.0233,  0.0390, -0.0126],\n",
       "                      [-0.0066,  0.0072, -0.0055,  ..., -0.0140,  0.0132, -0.0233],\n",
       "                      ...,\n",
       "                      [ 0.0176, -0.0025, -0.0217,  ...,  0.0282, -0.0322, -0.0053],\n",
       "                      [-0.0234,  0.0315,  0.0259,  ...,  0.0157,  0.0003,  0.0220],\n",
       "                      [-0.0019, -0.0255,  0.0138,  ..., -0.0202,  0.0182, -0.0205]])),\n",
       "             ('transformer.h.15.mlp.dense_h_to_4h.bias',\n",
       "              tensor([-0.4502, -0.0336, -0.0706,  ..., -0.4607, -0.0345, -0.0545])),\n",
       "             ('transformer.h.15.mlp.dense_4h_to_h.weight',\n",
       "              tensor([[-0.0168,  0.0095,  0.0009,  ...,  0.0373, -0.0182, -0.0093],\n",
       "                      [-0.0074, -0.0107,  0.0089,  ..., -0.0137, -0.0180,  0.0308],\n",
       "                      [ 0.0217, -0.0056,  0.0321,  ..., -0.0297,  0.0064, -0.0087],\n",
       "                      ...,\n",
       "                      [-0.0024, -0.0393,  0.0048,  ..., -0.0020,  0.0017, -0.0147],\n",
       "                      [-0.0213, -0.0251, -0.0257,  ..., -0.0166, -0.0075, -0.0021],\n",
       "                      [ 0.0109, -0.0050, -0.0071,  ..., -0.0032, -0.0046,  0.0168]])),\n",
       "             ('transformer.h.15.mlp.dense_4h_to_h.bias',\n",
       "              tensor([ 0.0740, -0.1770, -0.1487,  ...,  0.0006,  0.1949, -0.0283])),\n",
       "             ('transformer.h.16.input_layernorm.weight',\n",
       "              tensor([1.4043, 1.7275, 1.4365,  ..., 1.6172, 1.7354, 1.2793])),\n",
       "             ('transformer.h.16.input_layernorm.bias',\n",
       "              tensor([ 0.0159,  0.0354,  0.0161,  ...,  0.2391,  0.0714, -0.0463])),\n",
       "             ('transformer.h.16.self_attention.query_key_value.weight',\n",
       "              tensor([[ 7.7772e-04, -2.4048e-02, -2.2144e-03,  ...,  5.1081e-05,\n",
       "                       -2.3727e-02,  5.0163e-03],\n",
       "                      [ 1.3876e-03,  2.1835e-02,  2.5360e-02,  ...,  5.2460e-02,\n",
       "                       -2.5349e-03,  1.7075e-02],\n",
       "                      [ 4.2877e-03,  1.1429e-02, -2.0889e-02,  ...,  5.0316e-03,\n",
       "                        2.6993e-02, -1.8600e-02],\n",
       "                      ...,\n",
       "                      [ 2.9648e-02, -7.2479e-03,  2.8473e-02,  ...,  1.6571e-02,\n",
       "                       -2.2583e-02, -4.2023e-02],\n",
       "                      [ 2.3499e-02, -7.1487e-03,  1.4267e-02,  ..., -1.0773e-02,\n",
       "                       -3.2864e-03,  3.2806e-02],\n",
       "                      [ 2.0676e-02, -6.9809e-03, -1.4252e-02,  ..., -5.5237e-03,\n",
       "                       -1.0803e-02, -1.9241e-02]])),\n",
       "             ('transformer.h.16.self_attention.query_key_value.bias',\n",
       "              tensor([ 9.3323e-02,  1.4709e-01,  1.9714e-01, -2.8641e-02, -2.1610e-03,\n",
       "                      -2.4323e-02, -4.7577e-02, -2.7084e-03,  4.8248e-02,  5.4245e-03,\n",
       "                       8.7341e-02, -7.0068e-02,  1.3818e-01, -2.5970e-02,  6.2012e-02,\n",
       "                       1.0553e-01,  5.9235e-02,  1.3086e-01, -2.3010e-01,  1.4648e-01,\n",
       "                       4.8535e-01, -8.1177e-02,  8.0994e-02,  4.4220e-02,  2.3352e-01,\n",
       "                      -4.2816e-02, -8.6853e-02, -1.6143e+00,  7.9529e-02,  6.1340e-02,\n",
       "                      -8.5327e-02,  2.0984e-01, -1.8384e-01, -1.4380e-01, -1.3171e-01,\n",
       "                       1.5671e-02, -2.7435e-02, -7.5867e-02,  2.1436e-01, -1.6064e-01,\n",
       "                      -1.0095e-01,  9.8389e-02,  1.1633e-01,  3.4668e-02, -8.2825e-02,\n",
       "                       5.6458e-02,  7.3547e-02, -2.9736e-03, -7.2693e-02, -5.1086e-02,\n",
       "                       2.8003e-01,  9.8633e-02,  1.3208e-01,  5.0720e-02, -3.5248e-02,\n",
       "                      -1.5144e-02,  7.6782e-02,  1.4185e-01,  1.1368e-02, -1.4844e-01,\n",
       "                      -3.9490e-02,  1.3245e-01,  8.7708e-02, -2.6230e-02,  4.6570e-02,\n",
       "                       2.6562e-01,  3.9746e-01,  5.1819e-02, -3.3813e-02, -7.6942e-03,\n",
       "                      -7.8964e-04,  1.0339e-01,  5.0087e-03, -8.8562e-02,  9.5654e-01,\n",
       "                       1.3574e-01,  2.5558e-02, -3.8635e-02,  8.9355e-02,  2.4872e-03,\n",
       "                      -1.6479e-02, -2.1301e-02,  4.8218e-02,  1.5906e-01,  1.4153e-03,\n",
       "                      -5.8496e-01,  8.5632e-02,  6.9656e-03,  1.6150e-01,  1.3257e-01,\n",
       "                       8.3191e-02, -1.5579e-02,  8.1970e-02,  5.5084e-02,  1.8799e-02,\n",
       "                       1.1377e-01, -9.7885e-03, -1.0303e-01,  7.5569e-03, -1.3763e-02,\n",
       "                       4.9469e-02,  2.2659e-03, -6.0242e-02, -3.6163e-02,  6.1523e-02,\n",
       "                       1.5579e-02,  2.6169e-02, -3.0289e-03,  1.9699e-02,  1.3237e-02,\n",
       "                      -1.5271e-01, -4.5776e-03, -7.2632e-02,  2.2913e-01, -1.7456e-02,\n",
       "                      -1.0529e-01,  6.3049e-02, -1.5503e-01, -6.5491e-02, -8.7463e-02,\n",
       "                      -8.4595e-02, -9.4189e-01,  6.0547e-02, -6.2439e-02,  2.0237e-03,\n",
       "                      -2.9678e-02,  2.7504e-03,  1.1786e-01,  2.0056e-01, -2.0959e-01,\n",
       "                       1.2354e+00,  4.1528e-01, -2.4329e-01, -4.0820e-01, -2.3645e-01,\n",
       "                      -1.9788e-01,  1.5259e-01,  1.8506e-01,  1.6760e-01,  4.4098e-02,\n",
       "                      -3.4448e-01, -4.4434e-01, -3.7305e-01, -3.6035e-01, -4.7974e-01,\n",
       "                       1.3680e-02,  2.2500e+00,  2.5406e-02, -3.6768e-01, -1.6162e-01,\n",
       "                       3.1494e-01, -2.0050e-02,  3.8037e-01, -3.5889e-01,  2.3462e-01,\n",
       "                      -1.2311e-01,  1.6394e-01, -1.2659e-01,  2.4939e-01, -1.8884e-01,\n",
       "                      -1.0083e-01, -2.4902e-01, -8.0273e-01, -5.4346e-01,  3.0469e-01,\n",
       "                      -4.4653e-01, -2.2864e-01,  2.4316e-01,  1.1414e-01,  2.6221e-01,\n",
       "                      -1.7685e-02, -2.0251e-01,  1.9141e-01, -1.7656e+00, -5.9082e-01,\n",
       "                      -6.2561e-02,  1.4905e-01,  3.9673e-01, -1.2680e-02,  3.2251e-01,\n",
       "                       3.0786e-01,  1.5857e-01, -4.0698e-01, -7.9163e-02, -1.5405e-01,\n",
       "                      -2.0239e-01,  2.2156e-02,  4.1064e-01, -2.8906e-01,  7.6370e-03,\n",
       "                       3.8916e-01,  2.5610e-01,  8.5510e-02, -1.3525e-01,  3.2983e-01,\n",
       "                       1.5710e-01, -3.3179e-01, -1.4856e-01,  2.0401e-02,  1.7441e-02,\n",
       "                      -2.3071e-01, -2.5977e-01,  4.0503e-01,  2.1826e-01,  4.7876e-01,\n",
       "                       2.8125e-01, -7.7026e-02, -1.5686e-02,  1.6675e-01,  2.6062e-02,\n",
       "                      -4.5929e-02,  4.1992e-02,  1.3147e-01, -3.2129e-01,  2.6074e-01,\n",
       "                       8.3984e-02, -1.4648e-01, -7.6027e-03, -3.1934e-01,  2.6636e-01,\n",
       "                       3.8696e-02,  7.5256e-02, -2.3477e+00,  6.7810e-02,  3.9673e-02,\n",
       "                       1.3586e-01, -1.5859e+00, -9.6191e-02, -3.3643e-01,  6.2598e-01,\n",
       "                      -1.7224e-01,  2.6904e-01, -1.1932e-01, -1.7990e-02,  2.5415e-01,\n",
       "                      -6.5820e-01, -5.1422e-03,  1.0107e-01, -3.9453e-01, -1.6614e-01,\n",
       "                      -1.3452e-01,  2.0447e-01,  9.9182e-02,  5.7861e-02,  3.5913e-01,\n",
       "                       1.2524e-01,  3.1201e-01, -3.0029e-01, -4.9438e-01,  5.9967e-02,\n",
       "                       2.0660e-02, -2.7197e-01,  9.7900e-02, -7.6416e-02,  4.4586e-02,\n",
       "                       5.9387e-02, -5.6580e-02, -1.4420e-03,  8.2626e-03,  1.6830e-02,\n",
       "                       2.1271e-02, -3.1372e-02,  3.9703e-02,  2.8000e-02, -9.6054e-03,\n",
       "                      -5.7793e-04, -1.6510e-02,  3.3913e-03, -9.8877e-03,  3.3051e-02,\n",
       "                      -9.6512e-03,  2.7725e-02, -1.0765e-02,  2.8534e-03, -1.5991e-02,\n",
       "                      -3.1143e-02, -7.6332e-03, -4.1229e-02,  1.6937e-02, -1.9331e-03,\n",
       "                      -2.9556e-02, -2.5803e-02, -4.8340e-02, -3.7048e-02, -8.8425e-03,\n",
       "                       4.3579e-02, -3.5889e-02, -3.1490e-03, -2.0580e-03,  4.4128e-02,\n",
       "                       2.4323e-02,  1.2901e-02, -8.8930e-04, -5.3894e-02,  3.8727e-02,\n",
       "                       8.9169e-05,  2.4033e-03,  1.9064e-03, -1.2384e-01,  1.6754e-02,\n",
       "                       3.8696e-02, -1.6327e-02,  4.2389e-02, -3.3722e-02, -3.5461e-02,\n",
       "                       1.0254e-01,  1.8631e-02,  5.7030e-03, -2.3537e-03,  4.7119e-02,\n",
       "                      -2.3682e-02, -9.7656e-03,  3.2166e-02, -1.3306e-02, -4.0161e-02,\n",
       "                      -2.0782e-02, -5.0842e-02, -1.9928e-02, -4.5959e-02, -1.0796e-02,\n",
       "                       1.0437e-02,  2.1713e-02,  4.4403e-03,  5.9906e-02,  3.0457e-02,\n",
       "                      -3.4058e-01,  5.3436e-02, -6.1226e-03, -2.9404e-02,  1.1826e-02,\n",
       "                      -1.2410e-04,  1.0460e-02, -2.3254e-02,  1.7365e-02, -1.1176e-04,\n",
       "                      -5.8174e-03, -2.5511e-04, -1.6203e-03,  2.5833e-02, -8.9188e-03,\n",
       "                      -5.9471e-03, -1.2489e-02,  1.5152e-02, -2.0065e-02, -4.7646e-03,\n",
       "                      -1.0925e-02, -1.6113e-02, -2.3712e-02, -1.1307e-02, -3.0727e-03,\n",
       "                       2.1408e-02,  3.4943e-02,  1.0567e-02, -8.0013e-04, -1.7807e-02,\n",
       "                       6.0501e-03,  3.3741e-03,  1.9104e-02,  5.2155e-02,  6.8069e-05,\n",
       "                      -4.9500e-02, -3.7766e-03, -1.5221e-02,  1.8951e-02, -1.6022e-02,\n",
       "                      -5.9204e-02, -8.3847e-03, -3.4210e-02, -4.6509e-02,  1.3573e-02,\n",
       "                      -1.9638e-02, -3.8422e-02, -3.0472e-02,  8.7585e-02,  8.8811e-06,\n",
       "                      -5.2917e-02, -4.3373e-03, -2.5360e-02,  3.7323e-02, -1.8600e-02,\n",
       "                       1.2219e-01,  1.7452e-03,  1.2962e-02, -8.4305e-03, -9.2891e+00,\n",
       "                       2.9160e+00, -5.0898e+00, -7.9932e-01, -2.9180e+00, -7.9961e+00,\n",
       "                      -1.9734e+01,  2.3125e+00, -4.9844e+00,  8.6953e+00,  2.3109e+01,\n",
       "                       1.7754e+00,  1.1898e+01,  1.6531e+01,  4.7305e+00,  9.5000e+00,\n",
       "                       8.1250e+00, -9.1797e+00, -4.3384e-01,  8.1094e+00,  3.6875e+00,\n",
       "                      -3.2031e+01,  1.0840e+00,  4.5586e+00, -7.9766e+00,  2.7109e+00,\n",
       "                      -1.4719e+01,  2.9023e+00,  8.3984e+00, -1.0648e+01, -3.6531e+01,\n",
       "                       1.8433e-02,  6.5977e+00,  4.7695e+00, -1.0016e+01, -7.4561e-01,\n",
       "                      -2.5605e+00,  1.6938e+01,  8.1641e+00, -1.5928e+00,  7.1094e+00,\n",
       "                       4.6055e+00,  2.8789e+00,  5.9766e+00,  1.8984e+01, -1.1320e+01,\n",
       "                      -2.5859e+00,  4.4922e+00,  3.3359e+00,  3.9238e+00,  2.7484e+01,\n",
       "                      -2.1934e+00,  6.4990e-01, -8.4180e-01,  2.8362e-03, -7.1875e+00,\n",
       "                       4.9414e+00,  9.1187e-02, -4.1250e+00, -7.6611e-01, -1.3887e+00,\n",
       "                       1.4180e+01,  6.0820e+00,  1.3848e+00,  3.4609e+00,  1.1846e+00,\n",
       "                       2.1277e-01, -4.3726e-01, -2.8496e+00,  2.2620e-01, -2.1738e+00,\n",
       "                      -2.6680e+00,  5.3008e+00, -1.8301e+00, -5.6250e-01,  4.0820e+00,\n",
       "                       2.2812e+00, -1.2324e+00,  2.3750e+00, -1.6045e+00, -3.9961e+00,\n",
       "                      -2.5332e+00, -1.6875e+00,  6.9531e-01,  4.1602e+00,  1.0078e+00,\n",
       "                       8.0029e-01,  5.5817e-02, -9.8096e-01, -2.3027e+00, -2.3066e+00,\n",
       "                       3.5625e+00,  9.3750e-01, -6.1172e+00,  2.0781e+00,  2.6152e+00,\n",
       "                      -1.1523e+00, -5.5176e-01, -4.8279e-02,  3.0020e+00,  2.0195e+00,\n",
       "                      -5.1562e-01,  3.0020e+00, -6.1035e-01, -3.2266e+00, -3.6670e-01,\n",
       "                      -4.0781e+00, -1.7639e-01,  1.0742e+01, -3.8105e+00,  9.7510e-01,\n",
       "                      -1.7764e+00, -1.0879e+00, -1.0449e+00,  5.5176e-01,  1.9893e+00,\n",
       "                      -2.9902e+00, -6.3110e-02, -1.3232e+00, -1.9971e+00, -3.6094e+00,\n",
       "                       2.9277e+00, -1.8467e+00, -4.7461e-01, -1.2529e+00, -3.5488e+00,\n",
       "                      -3.3691e+00, -1.1064e+00,  2.3499e-01,  2.9346e-01,  1.3977e-01,\n",
       "                       5.3894e-02, -3.1177e-01,  4.7646e-03,  1.1047e-01, -1.8811e-01,\n",
       "                       4.2432e-01, -1.1688e-01, -3.6060e-01,  1.7114e-01, -2.3301e+00,\n",
       "                       5.0087e-03, -4.6501e-03, -1.1865e-01,  7.8857e-02,  4.3243e-02,\n",
       "                       1.5610e-02,  1.9629e-01,  6.5735e-02,  2.8027e-01,  7.4158e-02,\n",
       "                       5.4626e-02,  7.4219e-02,  3.2788e-01, -4.0497e-02,  1.7559e+00,\n",
       "                      -4.2038e-03, -1.6089e-01, -2.6196e-01, -1.9897e-01, -1.4062e-01,\n",
       "                       2.4939e-01,  3.6304e-01, -6.1554e-02, -1.8237e-01, -1.2915e-01,\n",
       "                       1.0602e-01, -1.2830e-01, -3.3569e-02,  7.7820e-02,  1.0089e-01,\n",
       "                       1.1542e-01, -7.4844e-03,  5.4260e-02,  1.4026e-01, -6.0547e-01,\n",
       "                      -1.6327e-02,  2.6929e-01,  2.1393e-02,  1.6101e-01, -9.6985e-02,\n",
       "                      -1.8555e-01, -9.9670e-02, -6.1432e-02, -1.9519e-01,  4.2603e-02,\n",
       "                      -1.5308e-01,  6.4148e-02,  2.0000e+00,  3.3112e-02,  8.8882e-03,\n",
       "                       2.0044e-01,  1.1147e-02,  5.0195e-01, -4.3915e-02, -6.1249e-02,\n",
       "                      -1.7175e-01, -2.8613e-01, -5.1221e-01, -2.3145e-01,  3.0298e-01,\n",
       "                       2.4377e-01,  5.4053e-01,  2.0825e-01,  8.5205e-02,  1.8428e+00,\n",
       "                       5.4980e-01, -2.6074e-01, -4.2896e-01,  1.3440e-01,  2.0801e-01,\n",
       "                      -2.7075e-01,  3.2910e-01,  7.2327e-02, -4.5483e-01, -1.8640e-01,\n",
       "                       3.8574e-01,  1.2854e-01,  1.5723e-01, -3.1470e-01,  1.8152e-01,\n",
       "                      -1.0956e-01,  3.5132e-01,  3.4351e-01, -1.9104e-01, -2.9663e-01,\n",
       "                       7.8320e-01, -2.2363e-01, -2.9602e-03, -3.4644e-01,  4.4556e-02,\n",
       "                       2.5317e-01, -4.7791e-02, -6.0196e-03,  2.7148e-01,  2.0935e-01,\n",
       "                       2.3572e-01,  3.4009e-01,  1.9983e-01,  3.6182e-01,  2.3267e-01,\n",
       "                       1.4801e-02, -2.7002e-01,  4.7632e-01, -5.1318e-01, -5.0732e-01,\n",
       "                      -9.6008e-02,  2.2266e-01,  4.3335e-01,  5.6610e-03, -2.9761e-01,\n",
       "                       7.2803e-01,  3.3130e-01, -2.8882e-01,  1.7688e-01, -2.9219e+00,\n",
       "                       3.6488e-03,  2.9160e-02, -5.0598e-02, -3.1403e-02, -6.2027e-03,\n",
       "                      -9.6359e-03,  4.1168e-02,  4.7791e-02, -1.8829e-02, -1.1360e-02,\n",
       "                      -9.6588e-03,  4.6425e-03, -1.5381e-02,  6.2408e-03,  2.9327e-02,\n",
       "                       1.1463e-03, -3.5667e-03,  2.9846e-02, -2.5513e-02,  2.2614e-02,\n",
       "                      -2.5238e-02,  2.4445e-02,  6.6528e-03,  3.6316e-02,  4.5380e-02,\n",
       "                      -6.9702e-02, -3.9398e-02, -1.5656e-02,  6.0547e-02, -2.2919e-02,\n",
       "                      -6.9809e-04,  1.3723e-03,  5.3558e-03, -4.3152e-02,  3.6591e-02,\n",
       "                      -1.9241e-02,  1.0674e-02, -2.1103e-02, -3.4393e-02, -2.0340e-02,\n",
       "                       7.5912e-03,  3.5309e-02,  2.7115e-02, -1.0605e-02, -1.4248e-03,\n",
       "                      -2.9541e-02, -4.0894e-03,  1.0529e-02, -1.7285e-04,  7.2144e-02,\n",
       "                       6.1127e-02,  1.8478e-02,  2.5177e-02, -3.7201e-02,  1.5930e-02,\n",
       "                      -1.8005e-02, -1.4725e-02, -2.3941e-02, -8.6060e-03,  5.9692e-02,\n",
       "                       2.7847e-03, -5.5084e-03, -1.6830e-02,  2.0950e-02,  3.9902e-03,\n",
       "                      -5.2605e-03,  4.2343e-03,  9.9716e-03, -6.5674e-02, -9.2850e-03,\n",
       "                       5.0011e-03,  4.5105e-02,  1.4172e-01,  6.4148e-02,  5.8960e-02,\n",
       "                      -9.5444e-03,  1.1612e-02,  1.4896e-03,  1.1292e-03,  2.8275e-02,\n",
       "                       7.8430e-02,  8.2703e-02,  4.5654e-02, -1.5755e-03,  3.9917e-02,\n",
       "                       2.4231e-02, -1.0857e-02,  1.9485e-02,  4.4342e-02,  2.6340e-03,\n",
       "                       8.5876e-02,  5.5046e-03, -7.3120e-02,  4.2267e-03,  1.6602e-02,\n",
       "                      -7.4310e-03, -3.1235e-02, -3.0426e-02, -3.8166e-03,  1.4000e-02,\n",
       "                      -3.1342e-02,  4.8351e-04, -4.1351e-03, -5.5847e-02,  5.7281e-02,\n",
       "                      -2.2461e-02, -7.0557e-02,  4.9133e-03, -4.6570e-02, -3.4515e-02,\n",
       "                       6.5369e-02, -3.9398e-02, -3.8940e-02,  2.6474e-02, -7.9422e-03,\n",
       "                       5.7251e-02,  1.7502e-02,  7.1144e-03, -3.4851e-02, -1.1871e-02,\n",
       "                       3.4790e-03, -2.1652e-02, -2.1729e-02,  3.9764e-02, -4.7516e-02,\n",
       "                      -1.9882e-02, -5.2887e-02, -1.1032e-02])),\n",
       "             ('transformer.h.16.self_attention.dense.weight',\n",
       "              tensor([[-0.0252,  0.0020, -0.0185,  ...,  0.0308,  0.0170,  0.0203],\n",
       "                      [ 0.0146, -0.0194, -0.0134,  ...,  0.0051,  0.0020, -0.0031],\n",
       "                      [-0.0080, -0.0073,  0.0080,  ...,  0.0252,  0.0050, -0.0139],\n",
       "                      ...,\n",
       "                      [ 0.0131,  0.0072,  0.0178,  ...,  0.0534,  0.0275, -0.0122],\n",
       "                      [ 0.0111, -0.0053, -0.0222,  ..., -0.0270, -0.0133,  0.0091],\n",
       "                      [ 0.0098,  0.0170,  0.0060,  ..., -0.0125,  0.0163,  0.0033]])),\n",
       "             ('transformer.h.16.self_attention.dense.bias',\n",
       "              tensor([ 0.0233, -0.0061,  0.0215,  ..., -0.0912, -0.0902,  0.0381])),\n",
       "             ('transformer.h.16.post_attention_layernorm.weight',\n",
       "              tensor([1.2217, 1.4102, 1.2676,  ..., 1.1738, 1.4795, 1.0918])),\n",
       "             ('transformer.h.16.post_attention_layernorm.bias',\n",
       "              tensor([ 0.0498, -0.0348,  0.1427,  ...,  0.1821,  0.0525,  0.1903])),\n",
       "             ('transformer.h.16.mlp.dense_h_to_4h.weight',\n",
       "              tensor([[ 0.0216,  0.0135, -0.0128,  ...,  0.0352, -0.0328, -0.0118],\n",
       "                      [ 0.0097, -0.0094,  0.0278,  ...,  0.0168, -0.0016,  0.0280],\n",
       "                      [-0.0475,  0.0066, -0.0259,  ..., -0.0208, -0.0057,  0.0256],\n",
       "                      ...,\n",
       "                      [ 0.0171,  0.0070, -0.0257,  ...,  0.0042, -0.0160, -0.0129],\n",
       "                      [-0.0129, -0.0019, -0.0214,  ...,  0.0152, -0.0165,  0.0454],\n",
       "                      [-0.0165,  0.0160, -0.0045,  ...,  0.0104, -0.0147,  0.0081]])),\n",
       "             ('transformer.h.16.mlp.dense_h_to_4h.bias',\n",
       "              tensor([-0.3230, -0.5112, -0.5679,  ..., -0.1708, -0.1587, -0.3801])),\n",
       "             ('transformer.h.16.mlp.dense_4h_to_h.weight',\n",
       "              tensor([[-0.0019, -0.0128,  0.0070,  ..., -0.0164, -0.0151,  0.0145],\n",
       "                      [-0.0046,  0.0263,  0.0123,  ...,  0.0042,  0.0040, -0.0023],\n",
       "                      [-0.0069,  0.0242, -0.0173,  ...,  0.0173,  0.0074,  0.0130],\n",
       "                      ...,\n",
       "                      [ 0.0184,  0.0062,  0.0311,  ...,  0.0167, -0.0114, -0.0029],\n",
       "                      [-0.0126,  0.0028, -0.0302,  ..., -0.0021,  0.0045, -0.0109],\n",
       "                      [-0.0085,  0.0119,  0.0216,  ...,  0.0249,  0.0087,  0.0098]])),\n",
       "             ('transformer.h.16.mlp.dense_4h_to_h.bias',\n",
       "              tensor([ 0.0098, -0.1770, -0.0309,  ...,  0.1744,  0.2035, -0.0203])),\n",
       "             ('transformer.h.17.input_layernorm.weight',\n",
       "              tensor([1.3555, 1.6641, 1.3877,  ..., 1.4941, 1.6240, 1.2031])),\n",
       "             ('transformer.h.17.input_layernorm.bias',\n",
       "              tensor([ 0.0129,  0.0238,  0.0211,  ...,  0.3052,  0.0619, -0.0364])),\n",
       "             ('transformer.h.17.self_attention.query_key_value.weight',\n",
       "              tensor([[ 0.0058, -0.0029,  0.0079,  ...,  0.0087,  0.0009, -0.0086],\n",
       "                      [-0.0107,  0.0134, -0.0100,  ...,  0.0305,  0.0094,  0.0005],\n",
       "                      [-0.0474,  0.0042, -0.0215,  ...,  0.0250, -0.0003,  0.0028],\n",
       "                      ...,\n",
       "                      [-0.0127,  0.0163,  0.0217,  ..., -0.0019,  0.0126, -0.0091],\n",
       "                      [-0.0147, -0.0375, -0.0104,  ..., -0.0219, -0.0341, -0.0118],\n",
       "                      [ 0.0244, -0.0162, -0.0013,  ..., -0.0014,  0.0223,  0.0341]])),\n",
       "             ('transformer.h.17.self_attention.query_key_value.bias',\n",
       "              tensor([ 4.3274e-02,  3.1830e-02, -8.7830e-02,  3.3783e-02,  1.1616e-03,\n",
       "                       6.1493e-03,  9.2602e-04, -2.4323e-02, -9.8938e-02,  5.2246e-02,\n",
       "                      -2.4734e-02, -3.6316e-02,  1.5020e-03, -6.7261e-02,  4.8340e-02,\n",
       "                      -4.3579e-02, -4.0283e-03,  5.2063e-02, -3.1204e-02, -1.1528e-02,\n",
       "                       1.6998e-02, -2.0752e-02,  1.6205e-02,  1.6739e-02,  1.6571e-02,\n",
       "                       5.4657e-02,  1.2917e-02,  1.5083e-02,  2.8896e-03, -1.1353e-02,\n",
       "                      -3.7048e-02, -3.1799e-02,  1.7914e-02,  9.0485e-03, -1.3153e-02,\n",
       "                       9.6970e-03, -1.4328e-02,  5.1697e-02,  2.6321e-02,  2.4094e-02,\n",
       "                       4.3237e-01,  2.9678e-02,  8.1635e-03, -5.0926e-03, -5.4443e-02,\n",
       "                       1.2604e-02, -3.6255e-02,  1.1421e-02,  5.9509e-03, -5.0507e-02,\n",
       "                      -1.9043e-02, -1.9745e-02, -2.9984e-02,  1.2230e-02, -1.6006e-02,\n",
       "                      -5.7800e-02,  1.5266e-02, -1.4961e-02,  5.4413e-02, -3.1403e-02,\n",
       "                      -1.2482e-01,  6.4201e-03, -7.7057e-03,  1.6541e-02,  2.0654e-01,\n",
       "                      -6.4331e-02, -5.0049e-01,  1.4111e+00, -1.5247e-01, -2.0742e+00,\n",
       "                       1.3269e-01, -1.2744e-01, -2.4890e-01,  1.6006e+00, -1.6162e-01,\n",
       "                       1.8262e-01,  3.9886e-02, -2.9492e-01,  8.0566e-01, -6.6504e-01,\n",
       "                      -4.3481e-01, -1.7480e-01, -4.5874e-01,  1.4807e-01, -3.3667e-01,\n",
       "                       1.2138e-02, -3.4760e-02, -2.6685e-01, -6.2012e-01, -2.7783e-01,\n",
       "                      -6.7993e-02,  4.3701e-02,  1.0272e-01,  2.2314e-01,  3.5461e-02,\n",
       "                      -7.4806e-03, -1.0181e-01,  2.5806e-01,  1.1481e-01,  1.0638e-01,\n",
       "                      -4.5319e-02,  2.9883e-01,  1.8665e-01,  3.2593e-01, -1.0870e-01,\n",
       "                      -3.7256e-01,  4.2236e-02,  3.9160e-01,  4.4287e-01, -1.8530e-01,\n",
       "                       1.1072e-01,  8.8867e-02, -1.5247e-01, -2.1045e-01,  1.3965e-01,\n",
       "                       5.6580e-02, -3.7305e-01, -5.4138e-02, -3.1616e-01, -5.8545e-01,\n",
       "                      -5.1318e-01, -6.6162e-01,  2.8857e-01, -1.2744e-01,  3.6035e-01,\n",
       "                      -4.8535e-01,  6.9678e-01, -4.9878e-01, -1.7426e-02, -5.8838e-01,\n",
       "                       1.7615e-01,  1.3892e-01,  3.1036e-02, -2.7563e-01,  8.7952e-02,\n",
       "                       2.1729e-01,  2.6152e+00, -8.7585e-03,  1.2433e-01, -8.9172e-02,\n",
       "                      -4.7058e-02, -8.3618e-03,  1.8945e-01, -2.2803e-01,  2.9551e+00,\n",
       "                       5.0079e-02, -1.2952e-01,  3.2129e-01, -1.8396e-01,  7.1228e-02,\n",
       "                       6.9238e-01, -2.6184e-02,  2.0117e-01,  1.8835e-01,  3.5107e-01,\n",
       "                       5.3418e-01, -1.6068e-02,  2.3560e-02, -6.6650e-02, -5.9753e-02,\n",
       "                       1.4526e-01, -7.0190e-02, -1.7688e-01, -1.9702e-01,  8.4961e-02,\n",
       "                      -9.4238e-02, -3.4888e-01, -3.9502e-01, -1.1237e-01,  2.0801e-01,\n",
       "                      -1.2720e-01, -7.4097e-02,  1.2329e-01, -6.6113e-01,  1.6553e-01,\n",
       "                      -6.6910e-03, -1.2549e-01, -2.4853e-03,  5.3329e-03, -1.4941e-01,\n",
       "                      -1.9946e-01, -2.5085e-02, -4.0009e-02, -1.9067e-01, -1.2573e-01,\n",
       "                      -1.0781e+00,  7.9285e-02, -3.3838e-01, -6.3049e-02,  5.7556e-02,\n",
       "                       5.6305e-03, -8.3557e-02, -7.1678e-03, -7.7515e-02,  2.8732e-02,\n",
       "                       2.9388e-02, -1.8661e-02, -2.9678e-02,  4.7516e-02,  3.4637e-02,\n",
       "                       2.9984e-02, -1.0262e-03, -1.0872e-02, -1.5808e-02,  4.0802e-02,\n",
       "                       1.0651e-02,  1.9348e-02, -1.6083e-02,  3.2928e-02,  6.7932e-02,\n",
       "                      -2.8030e-02,  5.5647e-04,  9.7580e-03,  4.0497e-02,  1.1253e-02,\n",
       "                      -2.1088e-02,  2.1042e-02,  5.3253e-02, -4.1504e-03, -1.3824e-02,\n",
       "                       5.8098e-03,  2.5444e-03,  4.2053e-02, -3.0426e-02, -2.8613e-01,\n",
       "                       6.0699e-02,  1.7139e-01,  4.2511e-02, -6.9336e-02,  5.0415e-02,\n",
       "                      -3.8544e-02,  2.7298e-02, -6.4758e-02,  2.5436e-02, -2.4891e-04,\n",
       "                      -1.4603e-02, -2.9495e-02, -7.5226e-03, -2.9022e-02, -4.8340e-02,\n",
       "                       2.5650e-02, -4.4250e-03,  3.7670e-03, -2.4704e-02,  3.5950e-02,\n",
       "                      -2.8305e-03,  3.6835e-02, -9.5825e-03, -3.0731e-02, -1.7838e-02,\n",
       "                       4.2267e-02, -1.7212e-02, -3.0243e-02,  4.5807e-02, -3.4821e-02,\n",
       "                       2.1210e-02,  2.4377e-01,  4.2920e-01, -1.1798e-01,  1.1169e-01,\n",
       "                      -5.0049e-01, -2.3984e+00, -7.4341e-02,  2.1106e-01,  1.5060e-02,\n",
       "                      -7.7942e-02,  4.3774e-01, -3.0273e-01,  1.1176e-01, -1.6736e-01,\n",
       "                      -1.1859e-01, -3.5107e-01, -7.9224e-02, -1.2286e-01, -4.8120e-01,\n",
       "                       3.6987e-01,  1.6418e-01, -2.5073e-01,  1.5588e-01,  5.4102e-01,\n",
       "                       3.2666e-01, -5.3809e-01,  4.3945e-01, -3.0396e-01,  4.8730e-01,\n",
       "                      -8.9600e-02,  6.1670e-01,  1.1420e-01,  1.6220e-02, -5.7910e-01,\n",
       "                      -3.0533e-02,  4.6582e-01, -2.5635e-02,  1.5930e-01,  1.6382e-01,\n",
       "                       4.8706e-01, -5.6738e-01, -1.5552e-01, -2.5293e-01,  4.4043e-01,\n",
       "                       2.7344e-01,  7.6050e-02, -9.7778e-02, -1.7944e-01,  1.2213e-01,\n",
       "                      -5.1367e-01,  4.0210e-01, -1.5833e-01,  2.2791e-01,  1.3013e-01,\n",
       "                      -4.4708e-02, -3.6914e-01, -2.3218e-01, -4.9512e-01,  2.8955e-01,\n",
       "                       1.2463e-01, -4.8560e-01,  1.0510e-01, -7.9895e-02, -2.4500e-01,\n",
       "                       1.8193e+00,  2.3457e+00, -2.6392e-01, -5.9961e-01, -4.0195e+00,\n",
       "                      -1.3734e+01, -2.7686e-01,  2.4766e+00,  6.4307e-01,  6.0156e-01,\n",
       "                       1.6826e+00, -1.6865e+00,  5.4443e-01,  5.6055e-01, -1.0605e+00,\n",
       "                      -3.2520e-01,  6.0730e-02, -5.8936e-01, -1.5049e+00,  1.5771e+00,\n",
       "                      -4.5435e-01,  2.1387e-01,  1.4814e+00,  9.7021e-01,  1.7761e-01,\n",
       "                      -3.4453e+00,  2.6445e+00, -1.4951e+00,  2.1172e+00, -1.5986e+00,\n",
       "                       2.1934e+00, -1.3750e+00, -1.2539e+00, -2.0918e+00, -5.8398e-01,\n",
       "                       2.1035e+00,  1.6699e-01,  2.1094e-01,  6.8359e-01,  1.9141e+00,\n",
       "                      -1.3857e+00, -1.3105e+00,  8.2031e-01,  2.6636e-01,  1.9421e-01,\n",
       "                       5.3418e-01, -3.2495e-01, -2.0273e+00,  4.5483e-01, -2.4988e-01,\n",
       "                       2.9316e+00, -1.0553e-01,  7.2949e-01, -5.4736e-01,  8.4912e-01,\n",
       "                      -6.8896e-01, -2.5415e-01,  1.1934e+00,  1.0352e+00, -1.6172e+00,\n",
       "                      -1.1348e+00,  1.1016e+00, -7.8174e-01, -1.0479e+00,  8.0185e-03,\n",
       "                       4.8950e-02, -1.7349e-02, -9.4376e-03, -2.9545e-03, -2.0325e-02,\n",
       "                      -2.3518e-03,  3.3813e-02, -1.6388e-02,  4.5105e-02,  9.9258e-03,\n",
       "                      -8.6823e-03,  4.6234e-03,  4.8340e-02, -1.1444e-03, -1.7883e-02,\n",
       "                       3.3936e-02,  3.8414e-03, -3.6987e-01, -7.7515e-02,  1.4450e-02,\n",
       "                       7.8125e-03,  1.6968e-02, -1.7490e-03, -7.7057e-03,  1.6586e-02,\n",
       "                      -2.9800e-02, -6.3181e-04,  1.3969e-02, -3.8788e-02,  1.0483e-02,\n",
       "                      -1.6556e-02,  3.4668e-02,  4.4861e-03,  3.5496e-03,  3.2684e-02,\n",
       "                      -1.6327e-02,  9.5520e-03, -1.5343e-02,  1.9547e-02, -2.4445e-02,\n",
       "                      -3.1433e-02,  2.7466e-02, -1.6251e-02,  1.2695e+00, -3.2623e-02,\n",
       "                      -1.6068e-02, -1.0361e-02,  3.1281e-02, -3.2654e-02,  1.0696e-02,\n",
       "                      -2.7206e-02, -1.0300e-02, -2.9640e-03,  1.1426e-01,  2.3102e-02,\n",
       "                      -5.1221e-01,  1.8433e-02,  2.5131e-02,  2.3300e-02,  4.1321e-02,\n",
       "                       1.6907e-02, -3.2654e-02, -1.7776e-02,  1.1029e-01,  1.0498e-02,\n",
       "                       2.8979e-01, -2.2327e-01,  2.4719e-01, -3.8940e-01,  3.4204e-01,\n",
       "                       5.3528e-02,  2.6685e-01,  1.5222e-01, -1.8921e-01, -1.6980e-01,\n",
       "                       3.6719e-01,  1.0455e-01, -3.4546e-02,  2.0618e-01, -8.4076e-03,\n",
       "                       8.6975e-02, -3.9502e-01,  9.7839e-02,  4.4629e-01, -6.5869e-01,\n",
       "                      -5.9180e-01,  1.1676e-01,  1.6296e-01, -1.5637e-01, -6.0303e-02,\n",
       "                       7.2632e-02, -4.9042e-02,  1.8340e+00,  2.8345e-01, -7.1240e-01,\n",
       "                       1.1975e-01,  2.2207e+00,  6.4404e-01, -6.6211e-01, -4.2139e-01,\n",
       "                       4.1504e-01,  5.3223e-01,  1.6504e-01,  4.2261e-01, -3.9624e-01,\n",
       "                      -3.1299e-01, -1.7041e-01,  5.3613e-01,  1.0822e-01, -3.9771e-01,\n",
       "                      -4.9536e-01,  1.7310e-01,  2.8638e-01, -4.1235e-01, -7.1831e-03,\n",
       "                      -1.4908e-02, -4.5532e-01,  1.0248e-01, -2.8711e-01,  5.7434e-02,\n",
       "                       2.2351e-01, -2.3206e-01,  3.3618e-01, -2.9602e-02,  3.5156e-01,\n",
       "                      -3.2275e-01, -2.4395e-03,  2.2469e+01,  1.1357e+00, -4.9805e+00,\n",
       "                      -1.2168e+00,  2.4768e-01, -3.1621e+00, -1.5137e-01, -4.0454e-01,\n",
       "                       3.0723e+00, -1.0344e+01,  1.7297e-01, -9.6875e-01, -3.1914e+00,\n",
       "                       2.1113e+00,  3.1523e+00,  8.6406e+00,  1.4648e-01, -3.9980e+00,\n",
       "                       1.6582e+00, -3.2257e-02,  3.3145e+00,  2.5117e+00,  1.0586e+01,\n",
       "                      -3.0453e+01, -2.3457e+00, -2.2266e+00, -1.8584e+00,  8.8623e-01,\n",
       "                       3.2056e-01,  4.3555e-01,  2.3398e+00, -1.4473e+00,  4.5923e-01,\n",
       "                      -3.1484e+00,  1.3086e+00,  2.5586e+00, -2.4004e+00, -6.5479e-01,\n",
       "                       1.5068e+00,  3.9917e-01,  3.1855e+00,  1.6592e+00, -3.6621e-01,\n",
       "                      -4.1455e-01, -4.1309e-01,  1.5635e+00,  4.1953e+00, -5.7178e-01,\n",
       "                      -2.2598e+00, -9.6973e-01,  1.1602e+00, -2.8223e+00,  6.2891e-01,\n",
       "                       2.5610e-01, -1.0918e+00,  9.4873e-01, -4.4727e+00,  3.0000e+00,\n",
       "                       4.4751e-01,  7.3389e-01, -5.8057e-01, -2.6328e+00,  4.9141e+00,\n",
       "                       6.0840e-01,  2.9068e-02, -3.9551e-02,  1.4915e-03,  3.3600e-02,\n",
       "                      -1.5610e-02, -4.9805e-02,  2.5024e-02,  1.3638e-03, -1.6602e-02,\n",
       "                      -1.1200e-02,  1.1543e-02,  5.0507e-02,  6.0730e-03,  1.0300e-02,\n",
       "                       4.5471e-03, -9.8133e-04, -3.0457e-02,  7.5188e-03, -1.3153e-02,\n",
       "                      -9.5291e-03,  5.6267e-04, -4.4365e-03, -3.1891e-02, -2.8046e-02,\n",
       "                      -8.3494e-04,  1.6113e-02,  1.1909e-02,  3.3752e-02, -3.4302e-02,\n",
       "                      -2.3666e-02,  1.1864e-02, -3.2013e-02,  1.7672e-03,  2.0187e-02,\n",
       "                      -7.6485e-04, -2.1317e-02,  9.6436e-03,  1.3794e-02, -1.4906e-03,\n",
       "                       7.7667e-03, -3.4302e-02, -2.4750e-02, -3.0499e-03,  9.0637e-03,\n",
       "                      -2.4887e-02,  2.6413e-02, -6.2103e-03,  1.7578e-02,  8.7967e-03,\n",
       "                      -1.5671e-02,  5.3070e-02, -1.2024e-02, -4.0497e-02, -3.5057e-03,\n",
       "                      -9.1858e-03, -1.5144e-02, -1.7502e-02,  3.0930e-02,  7.2556e-03,\n",
       "                       3.8635e-02,  1.4229e-02,  7.8087e-03,  1.8127e-02, -1.7914e-02,\n",
       "                       9.8828e-01,  5.2100e-01,  1.5540e-01,  8.6975e-02, -5.0781e-02,\n",
       "                       6.7920e-01,  5.2588e-01,  2.7466e-02, -1.5625e-01, -5.4443e-01,\n",
       "                       2.2539e+00, -1.9397e-01,  7.7209e-02, -1.5833e-01,  4.1113e-01,\n",
       "                      -6.1523e-01,  1.9238e-01, -1.6162e-01, -6.0352e-01,  9.8896e-04,\n",
       "                       1.3586e-01,  5.4993e-02, -6.4453e-02, -1.8707e-02,  3.5938e-01,\n",
       "                      -1.2866e-01,  6.0254e-01,  1.2158e-01,  5.2100e-01,  5.5420e-01,\n",
       "                       4.8859e-02, -4.6216e-01, -4.6338e-01,  5.2979e-01, -1.1218e-01,\n",
       "                       5.0635e-01,  6.5137e-01,  2.0859e+00, -5.6348e-01,  4.1187e-01,\n",
       "                      -3.4515e-02,  4.4861e-02,  1.6162e-01, -5.4291e-02, -2.0435e-01,\n",
       "                      -3.0713e-01, -8.2336e-02,  1.0333e-01, -4.2896e-01,  4.6216e-01,\n",
       "                      -1.0645e-01, -1.2646e-01,  1.9043e-01,  1.4099e-01, -4.9365e-01,\n",
       "                       2.0544e-01,  3.9160e-01,  2.4072e-01,  1.7358e-01,  2.5487e-04,\n",
       "                       7.6721e-02,  4.9164e-02,  2.2351e-01, -4.0771e-01, -6.5156e+00,\n",
       "                      -2.9082e+00,  2.8926e+00, -1.4707e+00, -6.2422e+00, -3.1895e+00,\n",
       "                      -4.8789e+00, -7.5547e+00, -1.6357e-01,  7.2656e-01, -1.2047e+01,\n",
       "                       4.7930e+00, -3.2285e+00,  7.9102e-01,  3.3535e+00,  2.5703e+00,\n",
       "                       4.2539e+00, -4.0312e+00,  1.4668e+00, -2.1211e+00, -2.0938e+00,\n",
       "                      -2.6973e+00, -1.5088e+00, -5.5371e-01, -5.4736e-01, -4.4336e-01,\n",
       "                      -4.6509e-02, -1.3594e+00,  4.7266e+00, -8.9062e+00,  1.5698e-01,\n",
       "                       5.1641e+00, -4.7852e+00, -6.5430e+00,  1.0382e-01, -2.1797e+00,\n",
       "                      -3.6304e-01, -9.0234e+00,  2.9414e+00, -2.5269e-01,  7.7100e-01,\n",
       "                      -4.3008e+00,  5.5742e+00,  4.3711e+00,  8.9697e-01,  2.1016e+00,\n",
       "                       1.6123e+00, -1.8887e+00, -1.5820e+00, -6.0977e+00, -1.9160e+00,\n",
       "                       8.1787e-01,  2.6992e+00,  9.4092e-01, -1.0944e-01, -3.5859e+00,\n",
       "                       1.6367e+00,  2.8008e+00,  1.5361e+00,  1.4834e+00,  1.6387e+00,\n",
       "                       1.7832e+00, -1.8140e-01,  2.8086e+00])),\n",
       "             ('transformer.h.17.self_attention.dense.weight',\n",
       "              tensor([[-0.0171,  0.0119,  0.0006,  ...,  0.0042, -0.0010, -0.0125],\n",
       "                      [ 0.0012, -0.0183,  0.0116,  ...,  0.0027, -0.0042, -0.0018],\n",
       "                      [ 0.0014,  0.0039, -0.0311,  ...,  0.0044,  0.0083,  0.0088],\n",
       "                      ...,\n",
       "                      [-0.0085, -0.0245,  0.0269,  ..., -0.0045, -0.0032, -0.0009],\n",
       "                      [-0.0069, -0.0206,  0.0016,  ..., -0.0025, -0.0139, -0.0543],\n",
       "                      [ 0.0097, -0.0039, -0.0195,  ..., -0.0014,  0.0083, -0.0080]])),\n",
       "             ('transformer.h.17.self_attention.dense.bias',\n",
       "              tensor([-0.0497, -0.0103,  0.0540,  ..., -0.1177, -0.1560,  0.0235])),\n",
       "             ('transformer.h.17.post_attention_layernorm.weight',\n",
       "              tensor([1.3496, 1.4072, 1.2959,  ..., 1.0859, 1.4502, 1.1641])),\n",
       "             ('transformer.h.17.post_attention_layernorm.bias',\n",
       "              tensor([ 0.1405, -0.0195,  0.0790,  ...,  0.3696,  0.1074,  0.1175])),\n",
       "             ('transformer.h.17.mlp.dense_h_to_4h.weight',\n",
       "              tensor([[-0.0158,  0.0175,  0.0266,  ...,  0.0009, -0.0060,  0.0467],\n",
       "                      [ 0.0082,  0.0168, -0.0513,  ...,  0.0280, -0.0034,  0.0059],\n",
       "                      [-0.0184, -0.0027,  0.0006,  ...,  0.0073,  0.0235,  0.0048],\n",
       "                      ...,\n",
       "                      [-0.0144,  0.0430,  0.0064,  ..., -0.0137, -0.0198, -0.0002],\n",
       "                      [-0.0048, -0.0141, -0.0211,  ...,  0.0047,  0.0050, -0.0040],\n",
       "                      [ 0.0226,  0.0175,  0.0496,  ...,  0.0369, -0.0432,  0.0463]])),\n",
       "             ('transformer.h.17.mlp.dense_h_to_4h.bias',\n",
       "              tensor([-0.1025, -0.4001,  0.0194,  ..., -0.3501,  0.0641, -0.5620])),\n",
       "             ('transformer.h.17.mlp.dense_4h_to_h.weight',\n",
       "              tensor([[-5.9624e-03, -2.8253e-05,  4.6844e-03,  ..., -2.7100e-02,\n",
       "                       -6.8283e-03, -1.1301e-03],\n",
       "                      [-3.7323e-02, -1.7303e-02, -1.4175e-02,  ...,  2.4414e-02,\n",
       "                        3.3173e-02, -5.0964e-02],\n",
       "                      [-1.9211e-02, -4.9248e-03, -2.5177e-02,  ..., -7.1869e-03,\n",
       "                       -3.2539e-03, -8.7128e-03],\n",
       "                      ...,\n",
       "                      [-9.7580e-03, -5.0751e-02,  6.4182e-04,  ..., -4.4403e-02,\n",
       "                       -1.5869e-02,  4.7852e-02],\n",
       "                      [-2.4551e-02, -4.8141e-03, -2.0370e-02,  ..., -4.9347e-02,\n",
       "                       -1.1253e-02, -5.6732e-02],\n",
       "                      [-1.3550e-02,  1.8600e-02, -4.0627e-03,  ...,  2.7122e-03,\n",
       "                        4.6997e-03,  2.1240e-02]])),\n",
       "             ('transformer.h.17.mlp.dense_4h_to_h.bias',\n",
       "              tensor([-0.0590, -0.0746, -0.1008,  ...,  0.4214,  0.1902,  0.0474])),\n",
       "             ('transformer.h.18.input_layernorm.weight',\n",
       "              tensor([1.4004, 1.5664, 1.5918,  ..., 1.3877, 1.5566, 1.4473])),\n",
       "             ('transformer.h.18.input_layernorm.bias',\n",
       "              tensor([ 0.0149,  0.0114, -0.0273,  ...,  0.4111,  0.0625, -0.0604])),\n",
       "             ('transformer.h.18.self_attention.query_key_value.weight',\n",
       "              tensor([[ 0.0094,  0.0126,  0.0005,  ...,  0.0094, -0.0312, -0.0008],\n",
       "                      [ 0.0043, -0.0063, -0.0098,  ...,  0.0002,  0.0159,  0.0055],\n",
       "                      [-0.0199, -0.0158,  0.0026,  ..., -0.0103, -0.0057,  0.0111],\n",
       "                      ...,\n",
       "                      [-0.0003,  0.0053, -0.0109,  ..., -0.0294,  0.0075,  0.0160],\n",
       "                      [-0.0377, -0.0072, -0.0108,  ..., -0.0414, -0.0110,  0.0026],\n",
       "                      [ 0.0178,  0.0040,  0.0060,  ...,  0.0170,  0.0035,  0.0158]])),\n",
       "             ('transformer.h.18.self_attention.query_key_value.bias',\n",
       "              tensor([ 8.4778e-02, -1.5213e-02, -9.2163e-03,  1.3696e-01,  4.9194e-02,\n",
       "                      -2.0599e-03, -1.5747e-02, -6.1768e-02,  2.1500e-02, -4.4189e-02,\n",
       "                       8.5693e-02, -3.9337e-02,  9.7046e-03,  6.8604e-02,  1.1768e-01,\n",
       "                       7.4272e-03, -3.0632e-03, -6.3820e-03, -1.5404e-02,  4.8523e-02,\n",
       "                      -1.0760e-01, -5.1727e-03,  9.2697e-03, -4.1565e-02,  8.0750e-02,\n",
       "                      -4.7760e-02,  6.9542e-03, -2.9709e-02, -1.8702e-03, -3.1464e-02,\n",
       "                      -1.2871e-02,  1.7380e-02, -9.1125e-02,  1.9321e-03,  4.6539e-02,\n",
       "                      -1.3086e-01, -3.4637e-03,  2.8717e-02, -8.9233e-02, -2.7695e-02,\n",
       "                       2.1881e-02,  1.4961e-02,  6.3538e-02, -2.8229e-02,  1.0187e-01,\n",
       "                       1.2329e-02, -8.6823e-03, -3.7262e-02, -5.2490e-02, -3.5858e-02,\n",
       "                      -3.9703e-02, -1.8433e-01,  1.1133e-01,  3.9764e-02,  1.3418e-03,\n",
       "                       1.2695e-02, -4.1687e-02, -1.1810e-01, -5.6366e-02,  3.0685e-02,\n",
       "                      -2.5360e-02,  5.7068e-02, -3.9734e-02,  7.6477e-02, -9.6863e-02,\n",
       "                       2.3303e-01, -1.3965e-01, -2.5098e-01,  3.6475e-01,  1.8921e-01,\n",
       "                       3.2837e-02, -9.4873e-01, -1.9043e-01,  6.3574e-01,  5.4053e-01,\n",
       "                       1.9507e-01, -4.1168e-02, -2.8735e-01,  1.1211e+00,  4.8047e-01,\n",
       "                      -1.6992e-01,  1.5881e-01, -7.8491e-02,  3.1616e-01, -4.9316e-01,\n",
       "                       1.0907e-01, -1.6492e-01, -1.2000e-01, -2.2131e-01,  3.1250e-01,\n",
       "                       3.7988e-01,  1.8933e-01,  2.3523e-01, -2.3181e-01,  4.0552e-01,\n",
       "                      -2.7856e-01,  2.7512e-02,  7.9712e-02,  1.8127e-01, -2.7783e-01,\n",
       "                       6.4990e-01,  1.5127e+00, -2.1326e-01, -2.4780e-01, -8.1055e-02,\n",
       "                       1.7834e-01, -1.9373e-01, -1.9102e+00,  2.0410e-01, -3.5083e-01,\n",
       "                      -4.5361e-01, -4.0674e-01,  7.5439e-01,  6.6528e-02,  4.5288e-02,\n",
       "                      -7.0703e-01, -4.6729e-01, -7.6465e-01,  1.9458e-01, -5.8203e-01,\n",
       "                      -1.3440e-01,  5.5908e-01, -9.9304e-02, -1.9092e-01, -1.7102e-01,\n",
       "                      -6.6553e-01,  4.5044e-01,  4.5825e-01, -3.0566e-01,  4.1748e-02,\n",
       "                      -2.8107e-02,  8.3984e-02,  7.7148e-02, -1.0291e-01,  2.1805e-02,\n",
       "                       3.6373e-03,  4.8065e-03, -6.5979e-02,  7.6477e-02, -1.0738e-03,\n",
       "                       4.0131e-02,  3.7201e-02,  2.5635e-01,  2.6798e-03, -4.2212e-01,\n",
       "                       3.4515e-02,  1.1029e-01,  2.0288e-01,  2.0032e-01, -6.9092e-02,\n",
       "                       1.5759e-01,  1.0791e-01,  9.0637e-02, -5.0415e-02,  5.2757e-03,\n",
       "                       4.2816e-02, -3.7506e-02, -2.1631e-01,  4.5227e-02, -1.0303e-01,\n",
       "                      -7.3967e-03,  6.2805e-02,  8.3923e-02,  2.3956e-02,  1.6882e-01,\n",
       "                       9.0332e-03,  7.3792e-02,  6.6406e-02, -6.7711e-04, -1.5125e-01,\n",
       "                       6.8420e-02,  2.4414e-02, -5.7404e-02,  1.0907e-01,  1.3074e-01,\n",
       "                      -1.6553e-01,  2.7252e-02, -1.0931e-01,  4.8523e-02, -1.1201e+00,\n",
       "                       1.7468e-01,  2.7832e-02,  9.1553e-02,  1.8347e-01,  5.4932e-02,\n",
       "                      -7.5134e-02,  9.4604e-02,  5.6580e-02, -4.2285e-01,  1.4087e-01,\n",
       "                       5.4901e-02,  2.3645e-01,  4.3518e-02,  3.4180e-02, -5.7800e-02,\n",
       "                       1.3176e-02, -2.8610e-02,  4.6921e-03,  1.2787e-02, -3.6743e-02,\n",
       "                       2.7390e-02,  4.2953e-03,  3.3783e-02, -1.4145e-02, -9.8022e-02,\n",
       "                      -6.6910e-03, -3.2288e-02, -4.2267e-02,  1.9943e-02,  1.5305e-02,\n",
       "                       6.0310e-03,  1.4238e-03, -5.5618e-03, -3.0785e-03, -1.8005e-02,\n",
       "                      -6.0150e-02,  2.7161e-03, -1.0559e-01, -3.2745e-02, -3.3813e-02,\n",
       "                       8.2214e-02, -6.2744e-02,  2.4429e-02,  1.0590e-02,  3.2082e-03,\n",
       "                       1.3741e-02, -2.9449e-03,  1.1072e-03,  4.4983e-02, -7.5054e-04,\n",
       "                      -4.1687e-02, -2.7863e-02, -1.5274e-02, -3.9703e-02,  3.4210e-02,\n",
       "                       1.6586e-02,  1.9302e-03,  2.2011e-03, -2.8763e-02, -7.1373e-03,\n",
       "                       2.8534e-02,  6.7932e-02,  4.3060e-02,  2.9956e-01, -1.5793e-02,\n",
       "                      -2.3651e-02,  2.6901e-02, -2.8168e-02,  4.0924e-02,  1.2917e-02,\n",
       "                       2.6779e-02,  3.1929e-03,  6.1188e-02,  2.7405e-02, -1.3168e-02,\n",
       "                      -3.0167e-02,  3.3813e-01,  9.1980e-02, -6.0303e-01,  1.6833e-01,\n",
       "                       4.2755e-02, -2.5940e-02,  1.7607e+00, -3.6768e-01, -1.8872e-01,\n",
       "                       1.1938e-01,  2.3547e-01, -5.4053e-01, -2.0098e+00,  7.0508e-01,\n",
       "                      -1.4519e-02,  2.2925e-01,  2.2070e-01, -1.7578e-01, -3.7134e-01,\n",
       "                       6.7236e-01, -5.7422e-01, -6.2109e-01, -2.0569e-02, -5.7831e-02,\n",
       "                       6.6797e-01,  2.7368e-01, -2.2736e-02, -2.4780e-01, -4.2725e-01,\n",
       "                       7.3975e-02,  1.0986e-01,  4.0161e-01,  3.9185e-01,  4.7388e-01,\n",
       "                       9.4421e-02, -6.5918e-01, -7.9727e-03,  2.1289e-01,  3.4814e-01,\n",
       "                      -3.8013e-01, -2.3499e-01, -1.4062e-01,  1.0098e+00, -2.2363e-01,\n",
       "                       4.4580e-01, -1.1493e-01, -7.8809e-01, -7.3975e-01, -7.2571e-02,\n",
       "                       3.5181e-01, -1.6931e-01, -1.5698e-01, -4.2163e-01,  9.4727e-02,\n",
       "                       3.4351e-01, -1.4929e-01, -5.3174e-01, -2.5220e-01,  9.2712e-02,\n",
       "                      -6.1676e-02, -1.5454e-01, -4.0308e-01,  1.6465e+00, -2.6660e-01,\n",
       "                      -1.4233e-01,  3.2812e-01, -2.3834e-02,  4.0619e-02, -1.6641e+00,\n",
       "                       1.5839e-02,  1.2170e-01, -6.0974e-02, -5.3925e-02, -1.7481e-03,\n",
       "                      -1.4783e-01,  4.8145e-01, -7.0984e-02,  3.3436e-03, -1.5369e-01,\n",
       "                      -6.8176e-02, -1.4124e-01, -7.5806e-02, -1.9788e-01,  1.0608e-01,\n",
       "                      -5.8545e-01, -2.7417e-01, -3.3936e-01, -1.0887e-02,  1.7896e-01,\n",
       "                       2.8473e-02, -1.9458e-01,  2.2363e-01,  1.7273e-01,  9.1187e-02,\n",
       "                       2.0789e-01,  1.8262e-01,  4.8401e-02,  1.8860e-01, -1.5527e+00,\n",
       "                      -1.5967e-01,  8.3923e-02,  6.2469e-02, -2.1895e+00,  1.6357e-01,\n",
       "                       1.9238e-01,  3.4717e-01, -4.1675e-01, -1.6577e-01, -3.1714e-01,\n",
       "                      -2.7390e-02,  2.5806e-01, -1.8457e-01,  3.8281e-01,  6.7566e-02,\n",
       "                      -2.6929e-01,  3.5767e-01,  3.1885e-01,  1.8274e-01, -7.8674e-02,\n",
       "                      -4.7913e-02,  1.7224e-01, -9.0515e-02,  1.7871e-01, -8.9600e-01,\n",
       "                       6.9434e-01, -6.0059e-02, -1.4832e-01, -4.1199e-02,  4.8120e-01,\n",
       "                      -1.8018e-01,  3.8867e-01, -1.1139e-01,  3.5767e-02,  4.1779e-02,\n",
       "                      -8.1543e-01,  5.4492e-01, -4.9634e-01, -1.1309e+00, -2.5146e-01,\n",
       "                       4.1357e-01, -8.9722e-02, -3.3423e-01, -1.2170e-01, -1.7163e-01,\n",
       "                      -9.2725e-01, -2.1326e-01, -9.7717e-02, -3.8843e-01, -2.7661e-01,\n",
       "                      -1.9519e-01, -6.1951e-02,  2.2107e-01, -9.5654e-01, -1.2482e-01,\n",
       "                      -7.9773e-02,  2.2925e-01, -1.7224e-01,  3.9087e-01, -1.1250e+00,\n",
       "                      -5.8398e-01, -1.3660e-01,  3.9331e-01,  1.0712e-01,  1.2002e+00,\n",
       "                       1.4758e-01,  2.3584e-01, -9.8193e-01, -7.7783e-01, -3.0933e-01,\n",
       "                      -1.6052e-01, -7.8467e-01,  2.5171e-01, -1.2314e+00,  4.4336e-01,\n",
       "                      -4.3823e-01,  9.1748e-01,  3.4717e-01, -4.3066e-01, -6.1493e-02,\n",
       "                      -4.1479e-01, -3.6865e-02, -2.3877e-01, -4.4922e-01, -3.6328e-01,\n",
       "                       3.7720e-01,  1.9629e-01,  7.9248e-01, -2.6172e-01,  2.5977e-01,\n",
       "                       1.9714e-01,  1.0869e+00, -2.4002e-02,  2.9392e-03, -8.2245e-03,\n",
       "                       1.5915e-02, -2.6321e-02,  2.1229e-03, -2.6276e-02, -2.4124e-02,\n",
       "                       3.1342e-02, -6.3438e-03,  1.9293e-03,  3.0426e-02, -8.6853e-02,\n",
       "                      -1.2512e-02,  1.3725e-02,  4.0619e-02, -3.2272e-03,  6.3049e-02,\n",
       "                      -1.3647e-03, -3.8818e-02,  6.4316e-03, -5.5618e-03,  6.5422e-03,\n",
       "                      -9.6817e-03, -1.0696e-02, -3.3035e-03, -1.2102e-03,  1.8692e-02,\n",
       "                       6.3553e-03, -2.4750e-02,  5.3291e-03, -1.6602e-02,  2.2751e-02,\n",
       "                      -4.0131e-02,  1.1803e-02, -3.5610e-03,  2.7863e-02,  1.6510e-02,\n",
       "                       2.2568e-02,  2.4902e-02, -3.2623e-02, -3.2562e-02, -7.6561e-03,\n",
       "                       8.5983e-03,  1.1337e-02,  2.6581e-02,  1.1887e-02,  2.8641e-02,\n",
       "                       7.4280e-02, -1.1414e-01,  2.9621e-03, -3.7689e-02,  7.8630e-04,\n",
       "                      -7.5417e-03, -7.0114e-03, -4.7340e-03,  5.8105e-02,  3.9856e-02,\n",
       "                      -7.2594e-03,  2.7943e-03, -1.2817e-02,  2.2614e-02, -6.2799e-04,\n",
       "                      -2.1988e-02,  1.1986e-02,  3.1372e-01,  3.2178e-01, -1.2354e-01,\n",
       "                       4.2816e-02, -1.9495e-01,  3.0605e+00, -3.8257e-01, -1.3086e-01,\n",
       "                       5.4688e-01, -4.0259e-01,  1.0406e-01, -2.4719e-01, -1.2505e-02,\n",
       "                      -7.8125e-01, -1.8402e-02, -1.3135e-01, -8.4326e-01,  4.0723e-01,\n",
       "                      -7.3669e-02,  1.5808e-02,  2.1436e-01, -2.7832e-01,  4.1162e-01,\n",
       "                       1.5796e-01,  1.2070e+00, -3.0078e-01, -2.2913e-01,  1.8445e-01,\n",
       "                      -6.0059e-02, -1.6235e-01, -4.2773e-01,  1.8591e-01, -1.0338e-02,\n",
       "                      -6.1188e-02,  2.3022e-01,  1.1467e-02,  2.1143e-01,  2.7075e-01,\n",
       "                      -1.0864e-02,  4.9683e-01, -1.1279e+00,  1.0620e-01,  3.4595e-01,\n",
       "                      -2.9126e-01, -4.7827e-01, -4.2090e-01,  3.7598e-01, -7.6172e-01,\n",
       "                      -3.3618e-01, -5.3467e-01, -1.3989e-01, -6.5918e-02,  1.2329e-01,\n",
       "                      -4.2822e-01,  5.6934e-01,  2.4329e-01,  4.7379e-03,  3.0527e+00,\n",
       "                       3.1421e-01,  6.9031e-02,  2.7461e+00,  7.5012e-02,  2.7539e-01,\n",
       "                       1.5466e-01, -7.7295e-01,  1.7129e+00, -5.7031e+00,  5.2031e+00,\n",
       "                       5.9922e+00, -2.7984e+01, -1.9082e+00,  8.3740e-01, -1.1279e+00,\n",
       "                       1.3311e+00, -3.2285e+00,  2.6016e+00, -6.8408e-01,  2.9219e+00,\n",
       "                       1.8564e+00,  6.0312e+00,  8.2344e+00, -2.8945e+00, -7.5049e-01,\n",
       "                      -2.6582e+00,  1.6182e+00,  2.8979e-01, -2.5566e+00, -2.3027e+00,\n",
       "                      -9.8828e+00,  1.0840e+00,  3.8242e+00,  1.0410e+00,  3.6602e+00,\n",
       "                       9.7510e-01,  4.8906e+00,  5.1055e+00, -3.4688e+00,  3.2031e+00,\n",
       "                       3.3008e-01, -2.7930e-01, -1.1493e-01,  2.8984e+00,  7.6367e-01,\n",
       "                      -1.4685e-01,  5.6953e+00, -1.7490e+00, -2.0176e+00,  5.3672e+00,\n",
       "                      -2.1660e+00,  3.8892e-01, -3.9160e+00,  3.9766e+00,  2.6270e+00,\n",
       "                       4.4688e+00,  2.4629e+00,  2.1406e+00,  1.6768e+00,  5.1406e+00,\n",
       "                      -5.9883e+00, -3.3508e-02, -1.6436e+00, -2.6891e+01, -7.9922e+00,\n",
       "                      -2.6797e+00, -2.9047e+01, -4.6055e+00, -4.4453e+00, -4.0820e+00,\n",
       "                      -5.4688e+00,  3.3301e+00,  1.4111e+00, -2.1270e+00, -6.8711e+00,\n",
       "                      -5.4883e+00, -1.6191e+00, -6.0312e+00,  1.7900e+00, -3.9902e+00,\n",
       "                      -2.3281e+00, -3.0645e+00, -8.9990e-01,  1.4375e+00,  2.2969e+00,\n",
       "                      -7.7295e-01,  1.4526e-01,  6.3184e-01,  5.1055e+00, -5.2031e+00,\n",
       "                      -2.8027e+00, -2.1328e+00, -4.6680e+00, -3.8438e+00, -4.7891e+00,\n",
       "                       1.8135e+00, -2.7617e+00, -3.9824e+00,  2.7344e+00,  7.1562e+00,\n",
       "                      -3.5586e+00, -2.3086e+00, -1.8418e+00, -1.8496e+00, -1.2822e+00,\n",
       "                       9.1162e-01, -6.6602e-01, -4.1758e+00, -2.2500e+00, -9.0576e-01,\n",
       "                      -4.2542e-02, -1.3701e+00,  1.8242e+00,  2.2168e+00,  3.4629e+00,\n",
       "                      -2.4648e+00, -1.2705e+00,  3.2129e+00,  6.5234e-01,  4.3008e+00,\n",
       "                      -3.1677e-02, -1.0439e+00, -1.3555e+00, -5.7715e-01,  6.8516e+00,\n",
       "                       9.0430e-01, -3.7793e+00, -1.1885e+00, -3.2207e+00,  1.5156e+00,\n",
       "                      -2.0215e+00, -1.2861e+00, -7.4072e-01,  3.1895e+00, -2.9205e-02,\n",
       "                      -3.0777e-02, -1.7532e-02,  3.9276e-02,  1.7349e-02, -3.2349e-02,\n",
       "                       2.8503e-02, -1.6534e-04, -6.4514e-02, -1.8021e-02,  2.5818e-02,\n",
       "                      -3.5919e-02,  7.9775e-04, -2.1881e-02,  2.2507e-02,  2.5082e-04,\n",
       "                      -1.0567e-02, -1.0956e-02,  2.4681e-03,  2.8244e-02, -1.9196e-02,\n",
       "                      -1.9012e-02,  2.8152e-02, -2.3453e-02,  1.5091e-02,  1.8060e-04,\n",
       "                       1.8982e-02,  9.0637e-03, -1.2012e-01, -2.5162e-02,  3.1796e-03,\n",
       "                       2.8915e-02,  4.7302e-02,  1.3596e-02,  2.6665e-03, -3.1097e-02,\n",
       "                      -1.4297e-02,  8.1711e-03, -4.6425e-03,  9.2697e-03, -6.5002e-03,\n",
       "                       3.9062e-02, -3.6316e-02, -8.2458e-02, -4.7150e-02, -3.8910e-02,\n",
       "                      -3.9001e-02,  4.3365e-02, -4.3213e-02,  7.9060e-04, -5.5145e-02,\n",
       "                       7.1655e-02,  9.0122e-04, -2.2919e-02, -4.0833e-02,  5.9738e-03,\n",
       "                       4.4464e-02,  3.3386e-02,  1.6373e-02,  9.1629e-03,  1.6711e-01,\n",
       "                       1.5015e-01, -8.8745e-02,  1.3542e-02,  1.1902e-01,  3.0225e-01,\n",
       "                      -6.4636e-02, -3.9062e-01, -5.7812e-01, -4.1089e-01, -1.7773e-01,\n",
       "                      -4.2456e-01, -1.6675e-01,  1.5674e-01,  5.7617e-01,  2.7637e-01,\n",
       "                       3.6401e-01, -5.4736e-01,  2.2632e-01,  2.1716e-01, -2.4796e-02,\n",
       "                      -5.2588e-01, -3.1641e-01, -1.1650e+00, -5.7471e-01, -2.8296e-01,\n",
       "                       5.9570e-02,  1.1847e-01, -1.0583e-01,  6.3354e-02,  1.8372e-01,\n",
       "                       1.6113e-02,  3.0811e-01,  1.0406e-01,  1.0779e-01, -5.2246e-01,\n",
       "                      -1.2573e-01, -2.6416e-01,  4.8608e-01, -1.2520e+00,  7.9834e-01,\n",
       "                       1.8457e-01, -1.0803e-01, -2.0825e-01,  2.8076e-01,  2.0312e-01,\n",
       "                      -4.2773e-01, -3.0518e-01, -8.9478e-02,  1.2646e-01, -3.6133e-01,\n",
       "                      -1.4478e-01,  5.9668e-01,  2.8906e-01,  2.8564e-01, -2.7173e-01,\n",
       "                       7.5830e-01, -5.2441e-01,  3.0176e-01,  4.5679e-01,  4.1699e-01,\n",
       "                       2.3594e+00, -4.3530e-01, -4.8486e-01, -2.3499e-01, -4.6899e-01,\n",
       "                       3.7891e-01, -7.9651e-03, -7.3779e-01, -3.0742e+00, -2.8281e+00,\n",
       "                       8.6523e-01,  6.5674e-01,  2.4492e+00, -9.6777e-01,  9.2896e-02,\n",
       "                      -1.4746e+00,  9.1248e-02, -2.0977e+00, -1.2969e+00, -1.1396e+00,\n",
       "                       2.1855e+00, -2.2188e+00,  5.3750e+00, -1.4990e+00,  2.2388e-01,\n",
       "                       9.1125e-02, -1.0577e-01, -3.4043e+00, -3.1973e+00, -1.8672e+00,\n",
       "                       2.9395e+00, -7.2852e-01,  1.8408e+00, -1.7998e+00, -4.1626e-01,\n",
       "                      -3.3320e+00, -1.1182e+00, -1.6709e+00, -1.2900e+00, -1.1152e+00,\n",
       "                      -5.4004e-01,  2.5059e+00,  9.9375e+00, -2.2930e+00, -5.6982e-01,\n",
       "                      -1.6357e+00, -2.7129e+00,  1.0625e+00, -1.6123e+00,  1.2285e+00,\n",
       "                      -2.1035e+00, -2.7891e+00,  6.9043e-01,  8.7988e-01, -1.2725e+00,\n",
       "                      -4.0938e+00,  5.4541e-01, -1.6494e+00, -1.7549e+00,  3.9746e-01,\n",
       "                      -6.2012e-01,  4.6562e+00, -1.7988e+00, -1.6025e+00, -1.2805e+01,\n",
       "                      -2.1895e+00,  1.5029e+00,  5.3555e+00,  3.2969e+00, -9.8779e-01,\n",
       "                      -4.7607e-01,  3.8544e-02, -3.2440e-02,  5.4688e-02,  2.9953e-02,\n",
       "                      -1.9394e-02, -2.0554e-02,  2.1164e-02, -2.1835e-02, -8.1558e-03,\n",
       "                       4.4098e-02, -9.8999e-02, -4.5685e-02,  3.8544e-02, -1.0063e-02,\n",
       "                       4.5441e-02,  5.1300e-02, -3.0014e-02, -4.4128e-02,  2.9312e-02,\n",
       "                       5.5145e-02,  1.9363e-02,  1.9287e-02,  1.3634e-02, -3.9703e-02,\n",
       "                       3.0350e-02, -3.6278e-03,  1.0857e-02, -5.4016e-03,  1.8692e-02,\n",
       "                      -3.9856e-02,  7.2266e-02,  3.2776e-02,  6.1722e-03,  2.6657e-02,\n",
       "                       7.4829e-02,  1.0635e-02, -4.7089e-02,  1.3962e-02,  2.3804e-02,\n",
       "                      -1.4809e-02,  1.9302e-02, -4.0833e-02, -2.0309e-02, -1.7105e-02,\n",
       "                       4.4464e-02, -3.0441e-02, -1.8860e-01, -1.2323e-01,  4.2908e-02,\n",
       "                      -1.0645e-01, -1.2589e-02, -5.0476e-02,  6.5491e-02, -2.3926e-02,\n",
       "                      -1.6174e-02,  7.4585e-02, -6.5430e-02, -5.9814e-02,  4.0680e-02,\n",
       "                       6.6650e-02, -3.1433e-02,  6.7810e-02, -3.8788e-02, -1.4145e-02])),\n",
       "             ('transformer.h.18.self_attention.dense.weight',\n",
       "              tensor([[ 0.0162,  0.0256, -0.0150,  ...,  0.0115, -0.0071, -0.0113],\n",
       "                      [-0.0272, -0.0035, -0.0024,  ...,  0.0179, -0.0122,  0.0073],\n",
       "                      [-0.0161,  0.0024,  0.0038,  ...,  0.0028,  0.0166, -0.0063],\n",
       "                      ...,\n",
       "                      [ 0.0284,  0.0448,  0.0035,  ..., -0.0214,  0.0441, -0.0179],\n",
       "                      [ 0.0014, -0.0022, -0.0048,  ...,  0.0089,  0.0188,  0.0075],\n",
       "                      [ 0.0074, -0.0179, -0.0020,  ..., -0.0032,  0.0086,  0.0015]])),\n",
       "             ('transformer.h.18.self_attention.dense.bias',\n",
       "              tensor([ 0.0298,  0.0447,  0.0324,  ..., -0.3840, -0.0195,  0.0464])),\n",
       "             ('transformer.h.18.post_attention_layernorm.weight',\n",
       "              tensor([1.3564, 1.4111, 1.3945,  ..., 1.0654, 1.4580, 1.2803])),\n",
       "             ('transformer.h.18.post_attention_layernorm.bias',\n",
       "              tensor([ 0.0885, -0.0154,  0.1024,  ...,  0.5801,  0.1478,  0.1516])),\n",
       "             ('transformer.h.18.mlp.dense_h_to_4h.weight',\n",
       "              tensor([[-0.0109,  0.0169, -0.0287,  ..., -0.0192, -0.0098, -0.0128],\n",
       "                      [-0.0318, -0.0131,  0.0184,  ...,  0.0135,  0.0352,  0.0023],\n",
       "                      [ 0.0530, -0.0247,  0.0232,  ...,  0.0016, -0.0080, -0.0114],\n",
       "                      ...,\n",
       "                      [-0.0207,  0.0216,  0.0027,  ..., -0.0018,  0.0037, -0.0096],\n",
       "                      [-0.0022,  0.0277, -0.0211,  ..., -0.0134,  0.0258,  0.0114],\n",
       "                      [ 0.0189, -0.0007, -0.0099,  ..., -0.0347, -0.0245, -0.0031]])),\n",
       "             ('transformer.h.18.mlp.dense_h_to_4h.bias',\n",
       "              tensor([-0.3472, -0.2230, -0.0707,  ..., -0.1232, -0.5161, -0.2017])),\n",
       "             ('transformer.h.18.mlp.dense_4h_to_h.weight',\n",
       "              tensor([[-0.0187, -0.0044, -0.0366,  ...,  0.0036,  0.0251, -0.0294],\n",
       "                      [-0.0135,  0.0014,  0.0161,  ...,  0.0214,  0.0284,  0.0094],\n",
       "                      [-0.0206, -0.0215, -0.0067,  ..., -0.0148, -0.0289, -0.0025],\n",
       "                      ...,\n",
       "                      [ 0.0362, -0.0033, -0.0270,  ..., -0.0308,  0.0026,  0.0060],\n",
       "                      [-0.0106, -0.0104,  0.0097,  ..., -0.0101,  0.0396,  0.0040],\n",
       "                      [-0.0099,  0.0112,  0.0182,  ..., -0.0072,  0.0005,  0.0039]])),\n",
       "             ('transformer.h.18.mlp.dense_4h_to_h.bias',\n",
       "              tensor([-0.0059, -0.0946, -0.1676,  ...,  0.5010,  0.2394, -0.0438])),\n",
       "             ('transformer.h.19.input_layernorm.weight',\n",
       "              tensor([1.4346, 1.5410, 1.6035,  ..., 1.1797, 1.5527, 1.6895])),\n",
       "             ('transformer.h.19.input_layernorm.bias',\n",
       "              tensor([ 0.0247,  0.0068,  0.0050,  ...,  0.4805,  0.0884, -0.0601])),\n",
       "             ('transformer.h.19.self_attention.query_key_value.weight',\n",
       "              tensor([[-0.0105, -0.0046, -0.0140,  ...,  0.0312,  0.0161, -0.0027],\n",
       "                      [ 0.0050, -0.0326,  0.0312,  ..., -0.0428, -0.0318, -0.0003],\n",
       "                      [ 0.0341, -0.0172, -0.0396,  ...,  0.0179, -0.0189,  0.0080],\n",
       "                      ...,\n",
       "                      [ 0.0073,  0.0046, -0.0094,  ...,  0.0128, -0.0104, -0.0317],\n",
       "                      [ 0.0055,  0.0234,  0.0032,  ...,  0.0180, -0.0126,  0.0090],\n",
       "                      [ 0.0104,  0.0099,  0.0003,  ..., -0.0140,  0.0005,  0.0132]])),\n",
       "             ('transformer.h.19.self_attention.query_key_value.bias',\n",
       "              tensor([ 1.9153e-01, -2.9785e-01,  6.1279e-01,  2.9053e-01, -3.3105e-01,\n",
       "                       1.9849e-01,  1.0339e-01, -2.4622e-01, -6.2891e-01,  1.6003e-01,\n",
       "                      -4.3970e-01, -4.1504e-02,  1.1969e-01,  2.1265e-01,  4.7437e-01,\n",
       "                       2.6001e-01,  1.6129e-02, -8.5205e-02, -2.2986e-01, -3.1982e-01,\n",
       "                       9.5520e-02,  6.5918e-02, -2.4268e-01,  5.8008e-01, -2.9810e-01,\n",
       "                       9.5276e-02,  6.3574e-01,  1.6455e-01, -7.7197e-01,  1.7322e-01,\n",
       "                       2.7173e-01, -8.1665e-02, -2.3962e-01, -5.0995e-02,  2.5366e-01,\n",
       "                      -8.2812e-01, -2.1008e-01, -6.0400e-01,  7.0984e-02,  1.1377e-01,\n",
       "                      -5.3418e-01,  3.8452e-02,  1.9971e-01, -1.1499e-01, -4.3945e-01,\n",
       "                       2.2070e-01, -7.1045e-02, -3.3667e-01, -5.0977e-01,  4.0991e-01,\n",
       "                      -1.2451e-02, -1.7676e+00, -6.2378e-02, -1.6431e-01, -2.3242e+00,\n",
       "                      -7.7295e-01, -7.2998e-02, -4.2993e-01,  2.3669e-01, -6.1621e-01,\n",
       "                      -4.1235e-01,  2.5366e-01, -2.4734e-02,  3.3301e-01, -7.5836e-03,\n",
       "                      -6.8436e-03, -1.2520e-02, -8.2092e-03, -2.2888e-03, -1.1740e-03,\n",
       "                      -2.9419e-02, -1.8799e-02,  3.2196e-02, -2.6321e-02,  8.4229e-02,\n",
       "                       4.1924e-03, -8.0948e-03, -4.9072e-02,  2.6970e-03, -1.6113e-02,\n",
       "                       2.8896e-03,  3.8867e-01, -1.8982e-02, -6.9962e-03, -3.2864e-03,\n",
       "                       3.4760e-02,  1.9958e-02,  1.0870e-01, -5.3528e-02, -6.9458e-02,\n",
       "                      -8.3237e-03,  4.2969e-02, -3.1738e-02, -1.5053e-02,  1.9943e-02,\n",
       "                      -2.9434e-02, -8.1940e-03,  1.6342e-02, -1.3733e-02, -6.6910e-03,\n",
       "                       2.6978e-02,  3.4760e-02,  3.5950e-02,  3.4668e-02,  4.3488e-02,\n",
       "                      -8.4839e-03, -3.5583e-02, -2.5597e-03,  7.2327e-03, -2.6917e-02,\n",
       "                       6.7787e-03, -2.0187e-02, -3.5126e-02, -4.9347e-02, -5.3741e-02,\n",
       "                       2.7496e-02, -3.2776e-02, -4.8126e-02,  2.8896e-03,  3.7720e-02,\n",
       "                       3.4790e-03,  1.5762e-02, -4.2305e-03, -3.6102e-02,  1.3680e-02,\n",
       "                      -3.1586e-02,  1.3680e-02,  3.3813e-02, -7.7209e-02, -3.5950e-02,\n",
       "                       4.8920e-02, -3.9307e-02, -8.6975e-02,  1.3892e-01,  2.1399e-01,\n",
       "                       4.4373e-02, -1.5251e-02,  1.0889e-01, -6.5491e-02, -8.3801e-02,\n",
       "                      -5.5359e-02,  1.9250e-01, -4.9377e-02, -1.2018e-01,  1.6040e-01,\n",
       "                      -2.6188e-03,  4.5135e-02, -4.2999e-02, -2.2156e-01,  1.3916e-01,\n",
       "                      -1.1346e-01, -5.9967e-02,  4.6783e-02, -1.0919e-01, -7.0557e-01,\n",
       "                       2.5830e-01,  1.4026e-01, -2.0966e-02,  3.8025e-02, -2.9282e-02,\n",
       "                      -1.5564e-01, -6.0498e-01,  8.6182e-02,  2.2232e-02, -9.6069e-02,\n",
       "                       1.3354e-01, -1.0848e-05,  1.6064e-01,  4.5258e-02, -7.3486e-02,\n",
       "                      -1.4575e-01, -9.2773e-02,  4.1138e-02,  1.4990e-01,  5.0476e-02,\n",
       "                       1.1182e-01,  1.3660e-01, -2.5589e-02,  3.4046e-03,  1.0126e-01,\n",
       "                       7.9163e-02, -4.3121e-02, -2.9712e-01,  6.6846e-01, -1.3477e-01,\n",
       "                       1.9730e-02, -8.8806e-02, -1.5686e-01,  3.3203e-02, -2.8122e-02,\n",
       "                       1.9012e-02,  7.2327e-02, -5.4102e-01,  5.8887e-01,  8.7158e-02,\n",
       "                      -9.3506e-02, -5.1971e-02, -2.3291e-01, -3.5229e-01, -1.8311e-02,\n",
       "                       7.3389e-01, -2.5073e-01, -9.7961e-02,  3.4985e-01, -4.4922e-02,\n",
       "                       2.3108e-01, -8.8989e-02, -2.4207e-01, -4.1809e-02,  4.5361e-01,\n",
       "                       2.3792e-01, -2.0728e-01, -4.2554e-01, -3.1281e-02, -4.1162e-01,\n",
       "                       1.2622e-01,  4.5874e-01,  2.3727e-02, -2.2498e-01, -2.0557e-01,\n",
       "                       8.2092e-02, -1.7883e-01, -2.5391e-02,  2.5391e-01, -3.9185e-01,\n",
       "                      -5.1575e-02,  3.6304e-01, -2.3682e-01,  1.4160e-01, -4.8193e-01,\n",
       "                      -4.0015e-01, -1.9604e-01, -2.5342e-01,  2.2070e-01,  2.4097e-01,\n",
       "                      -3.1555e-02, -6.1035e-01,  3.5339e-02,  3.1055e-01, -1.8323e-01,\n",
       "                      -1.8809e+00, -8.7939e-01,  2.6474e-03,  4.1846e-01, -4.5264e-01,\n",
       "                      -3.2861e-01, -2.8662e-01,  1.6504e+00, -2.8198e-01, -1.8835e-01,\n",
       "                      -3.6255e-01, -5.7373e-01, -2.7075e-01,  1.0537e+00,  2.7222e-02,\n",
       "                      -6.6650e-02,  1.8811e-01, -3.1958e-01, -3.0737e-01, -1.3477e+00,\n",
       "                       5.6982e-01, -9.5093e-02, -2.5708e-01, -1.5527e-01,  4.6802e-01,\n",
       "                      -2.3392e-02,  2.5439e-01,  2.0251e-01,  2.7368e-01,  1.1938e-01,\n",
       "                       3.4839e-01,  4.6631e-01,  1.5637e-01,  3.3374e-01, -3.8623e-01,\n",
       "                      -8.7128e-03,  8.6792e-02, -2.3059e-01,  2.6709e-01,  4.2236e-01,\n",
       "                      -7.7332e-02, -7.1106e-02,  1.7566e-01,  2.6587e-01, -4.0723e-01,\n",
       "                       1.5466e-01, -1.0400e-01, -2.4524e-01, -1.8555e+00,  2.2131e-01,\n",
       "                      -3.5938e-01,  6.1182e-01, -1.0272e-01, -1.9116e-01, -3.5248e-02,\n",
       "                      -2.9712e-01, -1.7859e-01,  1.2793e+00, -2.7783e-01, -5.8502e-02,\n",
       "                      -1.1200e-02, -4.5068e-01, -8.6487e-02, -2.5537e-01,  3.1689e-01,\n",
       "                      -1.1530e-01,  3.7183e-01, -5.1605e-02,  1.6833e-01, -3.5425e-01,\n",
       "                      -2.8564e-01,  1.6589e-01,  6.5247e-02, -2.7222e-01,  4.8981e-02,\n",
       "                      -8.5266e-02,  3.1201e-01, -1.2598e-01,  3.9526e-01,  1.1932e-02,\n",
       "                      -1.3574e-01, -2.5049e-01, -1.8396e-01,  7.0984e-02,  9.8022e-02,\n",
       "                       3.8116e-02,  2.1167e-01,  1.6510e-02, -1.9666e-01,  1.4633e-02,\n",
       "                      -1.0114e-01,  5.3375e-02,  3.8965e-01,  1.7468e-01, -1.7798e-01,\n",
       "                      -1.7822e-01, -8.5938e-01,  1.9849e-01,  1.1154e-02, -3.1396e-01,\n",
       "                       2.1326e-01,  2.1667e-01,  9.6436e-02,  5.7556e-02, -8.0139e-02,\n",
       "                       2.4280e-01, -2.4548e-01, -3.7479e-03,  4.8645e-02,  7.3120e-02,\n",
       "                       2.4609e-01,  1.1807e+00, -8.1543e-01, -1.1328e-01,  1.3457e+00,\n",
       "                       3.0640e-01, -3.5010e-01, -2.6196e-01, -1.4856e-01, -1.2756e-01,\n",
       "                       4.1351e-02, -3.1177e-01, -1.1548e-01, -8.7256e-01, -3.8623e-01,\n",
       "                       2.3352e-01, -3.2623e-02, -6.8237e-02,  2.1570e-01,  1.8701e-01,\n",
       "                      -9.9060e-02,  1.6575e-03,  2.5562e-01, -3.6133e+00,  2.8052e-01,\n",
       "                       7.3047e-01,  2.1204e-01, -4.6533e-01, -3.5217e-02,  4.4006e-02,\n",
       "                      -4.5337e-01, -2.7578e+00, -1.6074e+00, -1.2482e-01,  1.3214e-02,\n",
       "                      -1.8330e-03,  1.0948e-02, -2.3766e-03,  4.6158e-03, -4.5654e-02,\n",
       "                      -5.7564e-03,  4.5630e-01,  4.8889e-02,  6.2561e-02,  1.6388e-02,\n",
       "                      -6.9458e-02,  4.2389e-02, -7.7454e-02,  1.2836e-03,  6.6376e-03,\n",
       "                      -3.1830e-02,  1.6434e-02, -3.9124e-02,  1.1986e-02, -3.7384e-02,\n",
       "                      -1.2947e-02, -7.5684e-03, -4.9866e-02,  1.0614e-01, -4.3976e-02,\n",
       "                       3.7811e-02,  5.4382e-02,  5.4962e-02,  2.2537e-02, -6.4270e-02,\n",
       "                       8.2275e-02, -4.9255e-02,  4.6204e-02,  1.1520e-03,  2.1408e-02,\n",
       "                       1.1435e-03, -9.1736e-02,  2.2339e-02, -8.3237e-03, -7.2388e-02,\n",
       "                      -1.2512e-02,  5.8960e-02, -8.5831e-03, -7.6416e-02,  3.2318e-02,\n",
       "                       1.0605e-02, -3.0502e-02, -1.6937e-02, -4.2450e-02, -4.8157e-02,\n",
       "                       5.4047e-02, -7.1259e-03, -2.9785e-02,  3.1464e-02, -1.2184e-02,\n",
       "                       2.0844e-02, -1.7796e-03,  7.7629e-03,  2.3331e-02,  3.0365e-02,\n",
       "                      -1.3989e-01, -6.3538e-02,  2.4246e-02, -1.6891e+01, -2.9316e+00,\n",
       "                      -6.4697e-01, -1.9082e+00,  2.4561e-01,  1.2832e+00, -1.6123e+00,\n",
       "                       5.1914e+00,  1.7695e+00,  3.5498e-01, -4.1229e-02,  1.6738e+00,\n",
       "                      -2.6660e+00, -1.7393e+00, -4.9023e+00, -3.6309e+00, -4.5586e+00,\n",
       "                       1.1953e+00, -1.1211e+00, -2.3511e-01,  4.1328e+00, -1.3711e+00,\n",
       "                      -3.5000e+00,  1.2959e+00, -1.5986e+00, -8.5889e-01,  1.4541e+00,\n",
       "                      -3.3652e+00,  1.5996e+00,  2.8086e+00,  3.2773e+00, -6.7656e+00,\n",
       "                       3.6084e-01,  3.7773e+00,  4.5312e+00,  5.7812e+00, -4.4214e-01,\n",
       "                       8.8818e-01, -3.8535e+00, -3.5977e+00, -2.2446e-02,  7.4170e-01,\n",
       "                      -6.6797e+00, -2.0996e-01,  3.0742e+00, -2.3828e+00, -1.9766e+00,\n",
       "                       8.6328e-01, -4.0898e+00, -1.2529e+00,  4.1094e+00,  3.8013e-01,\n",
       "                       1.0320e+01,  9.3848e-01,  4.8789e+00,  7.6294e-02, -9.1172e+00,\n",
       "                       1.3389e+00, -4.8789e+00, -1.7783e+00, -4.7734e+00, -1.0527e+00,\n",
       "                      -1.8062e+01, -2.8965e+00, -6.2790e-03, -3.3997e-02, -1.0063e-02,\n",
       "                       7.2754e-02, -7.8857e-02, -3.3600e-02, -4.2450e-02,  3.9948e-02,\n",
       "                       3.9825e-02, -4.4830e-02,  1.1504e-05, -2.9190e-02,  3.4424e-02,\n",
       "                      -4.5715e-02, -4.4946e-01,  1.5503e-01,  1.6141e-04,  8.4106e-02,\n",
       "                      -1.3779e-02,  2.6798e-04, -1.4412e-02,  7.3242e-02,  1.6251e-02,\n",
       "                      -2.0966e-02,  4.4250e-02,  2.2827e-02,  7.3059e-02,  3.2684e-02,\n",
       "                       3.1769e-02,  1.9928e-02, -9.0881e-02, -2.7714e-03,  1.7519e-03,\n",
       "                       3.8269e-02,  1.7410e-02,  4.3823e-02,  1.7410e-02, -1.2398e-02,\n",
       "                       2.9755e-02, -9.4528e-03,  3.8177e-02,  3.2288e-02,  3.3539e-02,\n",
       "                      -3.9215e-02,  9.4528e-03, -3.9490e-02,  3.4302e-02,  6.2866e-02,\n",
       "                       5.4321e-02,  2.2049e-02, -1.6464e-02, -1.9531e-02,  1.7822e-02,\n",
       "                      -2.3209e-02,  1.8234e-02, -2.4734e-02,  1.0063e-02, -2.7817e-02,\n",
       "                       1.0968e-01, -7.1335e-03, -2.3178e-02, -3.7994e-02,  2.4853e-03,\n",
       "                      -6.2447e-03, -9.4629e-01,  6.6553e-01,  1.5801e+00,  2.5918e+00,\n",
       "                       1.7002e+00, -8.8806e-02,  1.2598e+00,  2.5801e+00,  1.2783e+00,\n",
       "                       2.8496e+00,  5.5322e-01, -2.6426e+00,  6.0107e-01,  1.3806e-01,\n",
       "                      -2.1582e+00,  1.0193e-01, -1.2275e+00,  1.5166e+00, -5.4541e-01,\n",
       "                      -1.8369e+00, -4.1016e-01,  1.8579e-01,  1.7373e+00, -2.0410e+00,\n",
       "                      -1.1367e+00,  1.5102e+01,  6.0547e-01, -2.0137e+00,  5.9033e-01,\n",
       "                       9.1309e-01, -1.6162e+00, -8.0273e-01, -3.5957e+00, -9.3652e-01,\n",
       "                       1.1221e+00, -1.4420e-02,  5.5469e-01,  7.4951e-01,  1.0986e+00,\n",
       "                       1.1670e+00,  2.8027e-01, -1.0713e+00, -2.9941e+00,  1.1891e+01,\n",
       "                      -5.6201e-01,  3.0449e+00,  8.4619e-01, -2.0918e+00, -1.4209e-01,\n",
       "                       2.2922e+01,  2.6172e+00,  5.7109e+00, -1.0844e+01, -1.0980e-01,\n",
       "                       2.2461e+00,  6.6943e-01, -1.3154e+00, -7.9805e+00,  4.0195e+00,\n",
       "                      -1.8867e+00, -1.7188e+00, -6.0242e-02, -1.5020e+00, -2.4016e+01,\n",
       "                       2.6685e-01, -3.8428e-01,  1.4688e+00,  1.9531e-01, -1.7334e-01,\n",
       "                      -9.6008e-02, -4.3945e-01,  1.0504e-01, -2.9956e-01,  1.5552e-01,\n",
       "                      -2.5781e-01, -1.0681e-01, -2.1953e+00, -4.2407e-01, -1.7932e-01,\n",
       "                      -1.8188e-01, -2.0117e+00,  3.5767e-01, -2.0044e-01,  1.1603e-01,\n",
       "                      -4.5435e-01,  5.0659e-02,  1.5637e-01, -1.2585e-01, -7.6538e-02,\n",
       "                      -4.7266e-01, -2.6147e-01,  3.4961e-01,  3.7671e-01,  4.0381e-01,\n",
       "                       5.5762e-01, -1.7407e-01, -3.4741e-01,  4.6045e-01,  1.1084e-01,\n",
       "                       2.8076e-02,  5.1819e-02, -1.3599e-01,  2.7515e-01, -3.1934e-01,\n",
       "                       8.3313e-02, -1.1450e-01, -4.4043e-01, -2.6196e-01,  4.6326e-02,\n",
       "                      -3.0249e-01, -2.0093e-01, -2.0312e-01, -1.5247e-01, -2.3041e-02,\n",
       "                      -3.1006e-01,  2.0935e-01, -3.6621e-01,  4.0967e-01,  8.1421e-02,\n",
       "                      -4.1284e-01,  6.4941e-01,  1.9373e-01, -4.8633e-01, -3.2593e-01,\n",
       "                      -2.5781e-01,  2.2803e-01, -1.8799e-01,  2.0532e-01,  1.0201e-02,\n",
       "                      -2.9770e-02,  5.4047e-02, -7.6866e-03,  9.0942e-03,  9.1858e-03,\n",
       "                       1.1879e-02, -1.0506e-02, -4.3152e-02, -5.3955e-02,  6.0516e-02,\n",
       "                       4.8737e-02, -2.7490e-01,  9.6436e-03, -3.7018e-02, -8.4152e-03,\n",
       "                       3.0228e-02,  1.8921e-02, -1.8402e-02, -1.9791e-02,  2.0248e-02,\n",
       "                       3.7506e-02,  2.8320e-02,  6.3354e-02, -7.8506e-03,  8.9836e-04,\n",
       "                      -8.7509e-03,  1.3962e-02,  1.0841e-02, -7.2556e-03,  1.7715e-02,\n",
       "                      -7.1182e-03,  1.5244e-02,  2.9541e-02, -5.6801e-03,  1.9577e-02,\n",
       "                      -3.2532e-02,  9.2316e-03, -4.8370e-02,  2.7023e-02, -5.8594e-02,\n",
       "                      -1.6907e-02, -1.9531e-02,  6.4430e-03, -3.4119e-02, -2.7344e-02,\n",
       "                       4.6448e-02,  2.4948e-02, -2.8076e-03,  5.8479e-03, -5.4047e-02,\n",
       "                       1.1597e-01, -2.9816e-02,  4.6722e-02, -1.0323e-02,  3.3169e-03,\n",
       "                       3.1372e-02, -6.1655e-04, -2.2253e-01,  4.7207e-04, -3.5400e-02,\n",
       "                       4.6844e-02, -4.2038e-03,  1.8341e-02])),\n",
       "             ('transformer.h.19.self_attention.dense.weight',\n",
       "              tensor([[-0.0055, -0.0010,  0.0022,  ...,  0.0152, -0.0127,  0.0133],\n",
       "                      [ 0.0195,  0.0003, -0.0179,  ..., -0.0309,  0.0255, -0.0010],\n",
       "                      [ 0.0002,  0.0195, -0.0127,  ..., -0.0058, -0.0044,  0.0090],\n",
       "                      ...,\n",
       "                      [ 0.0522, -0.0374, -0.0219,  ..., -0.0097, -0.0396,  0.0049],\n",
       "                      [-0.0031,  0.0188, -0.0032,  ...,  0.0161,  0.0003,  0.0217],\n",
       "                      [-0.0023, -0.0101, -0.0004,  ..., -0.0184,  0.0068,  0.0009]])),\n",
       "             ('transformer.h.19.self_attention.dense.bias',\n",
       "              tensor([-0.0842,  0.0263,  0.0218,  ..., -0.5713, -0.1008, -0.0159])),\n",
       "             ('transformer.h.19.post_attention_layernorm.weight',\n",
       "              tensor([1.4053, 1.4355, 1.4492,  ..., 0.9556, 1.4639, 1.3262])),\n",
       "             ('transformer.h.19.post_attention_layernorm.bias',\n",
       "              tensor([ 0.0861, -0.0703,  0.1034,  ...,  0.6582,  0.1796,  0.1866])),\n",
       "             ('transformer.h.19.mlp.dense_h_to_4h.weight',\n",
       "              tensor([[-0.0242,  0.0004,  0.0065,  ..., -0.0301,  0.0040, -0.0199],\n",
       "                      [-0.0137, -0.0137, -0.0238,  ...,  0.0173, -0.0102, -0.0236],\n",
       "                      [ 0.0093,  0.0315, -0.0301,  ...,  0.0115,  0.0382, -0.0028],\n",
       "                      ...,\n",
       "                      [ 0.0166, -0.0099, -0.0217,  ...,  0.0173, -0.0175,  0.0121],\n",
       "                      [ 0.0163, -0.0006, -0.0097,  ..., -0.0452, -0.0003,  0.0073],\n",
       "                      [ 0.0011, -0.0063,  0.0094,  ...,  0.0076,  0.0030,  0.0056]])),\n",
       "             ('transformer.h.19.mlp.dense_h_to_4h.bias',\n",
       "              tensor([-0.2571, -0.4133, -0.1749,  ..., -0.5474, -0.1931, -0.1160])),\n",
       "             ('transformer.h.19.mlp.dense_4h_to_h.weight',\n",
       "              tensor([[-0.0081, -0.0192, -0.0605,  ...,  0.0067, -0.0141,  0.0031],\n",
       "                      [ 0.0219, -0.0163,  0.0147,  ..., -0.0264,  0.0166, -0.0088],\n",
       "                      [ 0.0038,  0.0188,  0.0075,  ...,  0.0143,  0.0069,  0.0019],\n",
       "                      ...,\n",
       "                      [ 0.0136, -0.0076, -0.0331,  ...,  0.0273, -0.0199,  0.0382],\n",
       "                      [ 0.0232, -0.0589, -0.0242,  ...,  0.0617,  0.0194,  0.0195],\n",
       "                      [-0.0110,  0.0085, -0.0097,  ..., -0.0111,  0.0115,  0.0195]])),\n",
       "             ('transformer.h.19.mlp.dense_4h_to_h.bias',\n",
       "              tensor([ 0.0219, -0.0121, -0.0201,  ..., -1.0176,  0.2205, -0.1075])),\n",
       "             ('transformer.h.20.input_layernorm.weight',\n",
       "              tensor([1.5146, 1.6543, 1.6650,  ..., 0.7959, 1.5547, 1.7959])),\n",
       "             ('transformer.h.20.input_layernorm.bias',\n",
       "              tensor([ 1.6713e-04,  3.5248e-02,  6.5651e-03,  ...,  5.7471e-01,\n",
       "                       5.6366e-02, -4.4525e-02])),\n",
       "             ('transformer.h.20.self_attention.query_key_value.weight',\n",
       "              tensor([[-0.0105, -0.0135,  0.0126,  ...,  0.0233, -0.0113,  0.0058],\n",
       "                      [-0.0311, -0.0080,  0.0184,  ..., -0.0033, -0.0089, -0.0157],\n",
       "                      [ 0.0011,  0.0396,  0.0212,  ...,  0.0370,  0.0056,  0.0042],\n",
       "                      ...,\n",
       "                      [-0.0003,  0.0453,  0.0192,  ..., -0.0103,  0.0351,  0.0207],\n",
       "                      [ 0.0121, -0.0051,  0.0047,  ...,  0.0101,  0.0329,  0.0609],\n",
       "                      [ 0.0098,  0.0150,  0.0145,  ..., -0.0042,  0.0084,  0.0287]])),\n",
       "             ('transformer.h.20.self_attention.query_key_value.bias',\n",
       "              tensor([-0.0395, -0.0467, -0.5342,  ..., -0.0493, -0.0019,  0.0046])),\n",
       "             ('transformer.h.20.self_attention.dense.weight',\n",
       "              tensor([[ 0.0017,  0.0057, -0.0073,  ...,  0.0025, -0.0140, -0.0035],\n",
       "                      [ 0.0043, -0.0073, -0.0050,  ...,  0.0216,  0.0142, -0.0062],\n",
       "                      [ 0.0031, -0.0111,  0.0059,  ..., -0.0074,  0.0025, -0.0073],\n",
       "                      ...,\n",
       "                      [-0.0102,  0.1865, -0.0732,  ..., -0.0910, -0.0386, -0.0496],\n",
       "                      [-0.0067,  0.0096,  0.0059,  ...,  0.0088, -0.0028,  0.0120],\n",
       "                      [-0.0060, -0.0078,  0.0009,  ..., -0.0043, -0.0262, -0.0144]])),\n",
       "             ('transformer.h.20.self_attention.dense.bias',\n",
       "              tensor([ 0.0105, -0.0052, -0.0306,  ..., -2.7988, -0.0045,  0.0630])),\n",
       "             ('transformer.h.20.post_attention_layernorm.weight',\n",
       "              tensor([1.5361, 1.5342, 1.6025,  ..., 0.2837, 1.5752, 1.4531])),\n",
       "             ('transformer.h.20.post_attention_layernorm.bias',\n",
       "              tensor([-0.0173,  0.0533,  0.0439,  ...,  2.0938,  0.0883,  0.0903])),\n",
       "             ('transformer.h.20.mlp.dense_h_to_4h.weight',\n",
       "              tensor([[-0.0232, -0.0327, -0.0240,  ..., -0.0051, -0.0324, -0.0139],\n",
       "                      [-0.0097,  0.0081,  0.0010,  ..., -0.0204, -0.0072, -0.0208],\n",
       "                      [-0.0202,  0.0044, -0.0186,  ...,  0.0019,  0.0021,  0.0061],\n",
       "                      ...,\n",
       "                      [-0.0122, -0.0086, -0.0171,  ..., -0.0023,  0.0003, -0.0065],\n",
       "                      [-0.0016, -0.0225, -0.0006,  ..., -0.0161, -0.0240,  0.0086],\n",
       "                      [ 0.0197,  0.0155, -0.0012,  ..., -0.0098,  0.0102, -0.0257]])),\n",
       "             ('transformer.h.20.mlp.dense_h_to_4h.bias',\n",
       "              tensor([-0.2406, -0.2162, -0.0450,  ..., -0.1226, -0.2654, -0.1766])),\n",
       "             ('transformer.h.20.mlp.dense_4h_to_h.weight',\n",
       "              tensor([[-0.0033, -0.0113, -0.0174,  ...,  0.0097, -0.0199, -0.0295],\n",
       "                      [-0.0110,  0.0402, -0.0188,  ...,  0.0127, -0.0497, -0.0193],\n",
       "                      [-0.0024, -0.0117, -0.0122,  ...,  0.0009,  0.0090,  0.0239],\n",
       "                      ...,\n",
       "                      [ 0.0734,  0.0094, -0.0147,  ...,  0.0114,  0.0218, -0.0222],\n",
       "                      [ 0.0092,  0.0043,  0.0067,  ...,  0.0223,  0.0023, -0.0089],\n",
       "                      [-0.0047,  0.0074, -0.0072,  ..., -0.0122, -0.0092, -0.0058]])),\n",
       "             ('transformer.h.20.mlp.dense_4h_to_h.bias',\n",
       "              tensor([-0.1066,  0.0351, -0.0381,  ..., -1.7500,  0.1376, -0.1580])),\n",
       "             ('transformer.h.21.input_layernorm.weight',\n",
       "              tensor([1.8516, 1.8545, 1.9834,  ..., 0.3047, 1.8184, 2.0703])),\n",
       "             ('transformer.h.21.input_layernorm.bias',\n",
       "              tensor([ 0.0687,  0.0356,  0.0019,  ...,  0.8843,  0.0629, -0.0079])),\n",
       "             ('transformer.h.21.self_attention.query_key_value.weight',\n",
       "              tensor([[ 0.0218,  0.0091,  0.0187,  ..., -0.0582,  0.0155,  0.0256],\n",
       "                      [ 0.0028, -0.0349, -0.0248,  ..., -0.0227,  0.0093,  0.0142],\n",
       "                      [-0.0083,  0.0823, -0.0096,  ..., -0.0141, -0.0059, -0.0216],\n",
       "                      ...,\n",
       "                      [-0.0135, -0.0207,  0.0136,  ..., -0.0178, -0.0109, -0.0656],\n",
       "                      [-0.0181, -0.0157,  0.0243,  ..., -0.0191, -0.0368,  0.0079],\n",
       "                      [-0.0045,  0.0248, -0.0078,  ...,  0.0139,  0.0316, -0.0130]])),\n",
       "             ('transformer.h.21.self_attention.query_key_value.bias',\n",
       "              tensor([ 0.2573, -0.9609, -0.5684,  ...,  0.0889, -0.1114, -0.0547])),\n",
       "             ('transformer.h.21.self_attention.dense.weight',\n",
       "              tensor([[-1.1833e-02,  9.5520e-03, -7.3318e-03,  ...,  1.4275e-02,\n",
       "                       -1.1711e-02, -1.1330e-02],\n",
       "                      [-1.6571e-02,  1.5533e-04,  2.8496e-03,  ...,  2.8725e-03,\n",
       "                        1.0815e-03,  4.7989e-03],\n",
       "                      [-3.2444e-03, -6.8588e-03,  5.1651e-03,  ..., -1.2459e-02,\n",
       "                       -1.0742e-02, -8.4229e-03],\n",
       "                      ...,\n",
       "                      [-5.9387e-02, -8.9722e-02, -2.0493e-02,  ...,  6.0638e-02,\n",
       "                        1.6772e-01, -8.3740e-02],\n",
       "                      [ 2.2888e-03,  8.2064e-04, -2.5406e-03,  ..., -8.0261e-03,\n",
       "                        7.1335e-03, -9.8050e-05],\n",
       "                      [-3.5381e-04, -3.2482e-03,  5.2948e-03,  ..., -2.2552e-02,\n",
       "                        1.2222e-02, -4.4670e-03]])),\n",
       "             ('transformer.h.21.self_attention.dense.bias',\n",
       "              tensor([ 1.0002e-02, -3.0457e-02,  2.3384e-03,  ..., -4.4102e+00,\n",
       "                      -2.1530e-02,  3.0090e-02])),\n",
       "             ('transformer.h.21.post_attention_layernorm.weight',\n",
       "              tensor([2.3340, 2.4043, 2.3691,  ..., 0.3413, 2.3770, 2.1445])),\n",
       "             ('transformer.h.21.post_attention_layernorm.bias',\n",
       "              tensor([0.0917, 0.0167, 0.0567,  ..., 1.7012, 0.0225, 0.0746])),\n",
       "             ('transformer.h.21.mlp.dense_h_to_4h.weight',\n",
       "              tensor([[ 0.0268,  0.0207, -0.0074,  ...,  0.0028, -0.0016,  0.0078],\n",
       "                      [-0.0036,  0.0186, -0.0084,  ..., -0.0083,  0.0113, -0.0391],\n",
       "                      [ 0.0101, -0.0016,  0.0111,  ..., -0.0023,  0.0110, -0.0188],\n",
       "                      ...,\n",
       "                      [-0.0065, -0.0187,  0.0159,  ...,  0.0028,  0.0144, -0.0227],\n",
       "                      [-0.0235,  0.0029,  0.0238,  ..., -0.0032,  0.0144, -0.0436],\n",
       "                      [-0.0267,  0.0142,  0.0215,  ..., -0.0086,  0.0113,  0.0039]])),\n",
       "             ('transformer.h.21.mlp.dense_h_to_4h.bias',\n",
       "              tensor([ 0.0836, -0.1372, -0.2427,  ..., -0.3208, -0.0967, -0.1595])),\n",
       "             ('transformer.h.21.mlp.dense_4h_to_h.weight',\n",
       "              tensor([[-0.0281,  0.0007,  0.0048,  ..., -0.0308,  0.0070,  0.0395],\n",
       "                      [-0.0186, -0.0156,  0.0032,  ..., -0.0016,  0.0256,  0.0034],\n",
       "                      [ 0.0149, -0.0548,  0.0036,  ...,  0.0132, -0.0068, -0.0057],\n",
       "                      ...,\n",
       "                      [-0.0051,  0.0654,  0.1028,  ...,  0.0364,  0.0787,  0.1138],\n",
       "                      [ 0.0011,  0.0040,  0.0351,  ..., -0.0252, -0.0163, -0.0128],\n",
       "                      [-0.0026,  0.0113,  0.0074,  ...,  0.0187,  0.0214, -0.0112]])),\n",
       "             ('transformer.h.21.mlp.dense_4h_to_h.bias',\n",
       "              tensor([-0.0466, -0.0322,  0.0249,  ..., -2.7812,  0.0161, -0.1069])),\n",
       "             ('transformer.h.22.input_layernorm.weight',\n",
       "              tensor([2.8984, 3.1816, 3.2773,  ..., 0.3921, 3.1191, 3.1680])),\n",
       "             ('transformer.h.22.input_layernorm.bias',\n",
       "              tensor([0.0201, 0.0531, 0.0317,  ..., 2.1699, 0.0493, 0.0129])),\n",
       "             ('transformer.h.22.self_attention.query_key_value.weight',\n",
       "              tensor([[ 0.0184, -0.0042,  0.0112,  ..., -0.0018, -0.0079,  0.0030],\n",
       "                      [ 0.0020, -0.0502, -0.0077,  ...,  0.0152, -0.0271, -0.0100],\n",
       "                      [ 0.0250, -0.0160, -0.0392,  ..., -0.0017,  0.0399, -0.0113],\n",
       "                      ...,\n",
       "                      [-0.0235, -0.0223,  0.0073,  ..., -0.0110, -0.0188,  0.0059],\n",
       "                      [ 0.0094,  0.0384,  0.0117,  ..., -0.0012,  0.0471,  0.0107],\n",
       "                      [ 0.0006, -0.0149, -0.0143,  ...,  0.0046,  0.0140,  0.0285]])),\n",
       "             ('transformer.h.22.self_attention.query_key_value.bias',\n",
       "              tensor([-0.3545,  0.1101,  0.4214,  ...,  0.0065, -0.0334,  0.0223])),\n",
       "             ('transformer.h.22.self_attention.dense.weight',\n",
       "              tensor([[ 0.0036,  0.0067,  0.0011,  ...,  0.0021, -0.0088, -0.0025],\n",
       "                      [ 0.0050,  0.0068, -0.0013,  ...,  0.0061,  0.0058, -0.0003],\n",
       "                      [-0.0006,  0.0128,  0.0142,  ..., -0.0100, -0.0035,  0.0030],\n",
       "                      ...,\n",
       "                      [-0.0247, -0.0213, -0.0220,  ...,  0.1729, -0.0324,  0.0972],\n",
       "                      [ 0.0121, -0.0030,  0.0054,  ..., -0.0085, -0.0056, -0.0048],\n",
       "                      [-0.0115,  0.0033,  0.0153,  ..., -0.0104,  0.0023,  0.0074]])),\n",
       "             ('transformer.h.22.self_attention.dense.bias',\n",
       "              tensor([ 1.9775e-02, -2.2568e-02, -1.5678e-03,  ..., -4.7305e+00,\n",
       "                      -4.3755e-03,  4.2511e-02])),\n",
       "             ('transformer.h.22.post_attention_layernorm.weight',\n",
       "              tensor([3.8398, 3.9609, 3.9395,  ..., 1.0186, 4.0000, 3.6777])),\n",
       "             ('transformer.h.22.post_attention_layernorm.bias',\n",
       "              tensor([ 0.0478, -0.0751, -0.1052,  ...,  3.2168, -0.0798, -0.0619])),\n",
       "             ('transformer.h.22.mlp.dense_h_to_4h.weight',\n",
       "              tensor([[-0.0051,  0.0125,  0.0092,  ..., -0.0026,  0.0363,  0.0090],\n",
       "                      [ 0.0112, -0.0043, -0.0054,  ..., -0.0030, -0.0238, -0.0375],\n",
       "                      [ 0.0029, -0.0242,  0.0133,  ..., -0.0016,  0.0245,  0.0107],\n",
       "                      ...,\n",
       "                      [ 0.0052,  0.0037,  0.0041,  ..., -0.0010, -0.0109, -0.0754],\n",
       "                      [ 0.0063, -0.0420,  0.0185,  ..., -0.0030,  0.0180,  0.0066],\n",
       "                      [-0.0192, -0.0164,  0.0058,  ..., -0.0014,  0.0261,  0.0176]])),\n",
       "             ('transformer.h.22.mlp.dense_h_to_4h.bias',\n",
       "              tensor([-0.2308, -0.1782, -0.0978,  ..., -0.3032, -0.3950, -0.1332])),\n",
       "             ('transformer.h.22.mlp.dense_4h_to_h.weight',\n",
       "              tensor([[ 1.9913e-03,  1.4124e-03,  3.3836e-03,  ..., -3.6652e-02,\n",
       "                       -1.8143e-02, -4.6448e-02],\n",
       "                      [ 2.6760e-03, -5.7716e-03,  3.6316e-02,  ...,  1.8768e-02,\n",
       "                        2.5024e-02, -1.8616e-02],\n",
       "                      [-4.7913e-03, -1.0605e-02, -5.3833e-02,  ..., -2.5311e-03,\n",
       "                       -9.9640e-03,  9.3174e-04],\n",
       "                      ...,\n",
       "                      [-9.5947e-02,  3.7048e-02, -6.9336e-02,  ..., -1.9989e-02,\n",
       "                       -2.8286e-03, -3.1738e-02],\n",
       "                      [-5.9586e-03, -1.4259e-02,  2.9774e-03,  ..., -5.4512e-03,\n",
       "                        3.2837e-02, -2.9236e-02],\n",
       "                      [ 7.5579e-05, -2.2232e-02, -6.7932e-02,  ..., -2.6428e-02,\n",
       "                        1.2077e-02, -1.8738e-02]])),\n",
       "             ('transformer.h.22.mlp.dense_4h_to_h.bias',\n",
       "              tensor([-0.0155, -0.0953,  0.0172,  ..., -0.3699, -0.0052, -0.1332])),\n",
       "             ('transformer.h.23.input_layernorm.weight',\n",
       "              tensor([3.2793, 3.3848, 3.5625,  ..., 0.6841, 3.2402, 3.2832])),\n",
       "             ('transformer.h.23.input_layernorm.bias',\n",
       "              tensor([-6.1249e-02, -1.7853e-02,  4.7803e-05,  ...,  1.2246e+00,\n",
       "                       2.1301e-02, -9.0637e-03])),\n",
       "             ('transformer.h.23.self_attention.query_key_value.weight',\n",
       "              tensor([[ 0.0184,  0.0159,  0.0142,  ..., -0.0062,  0.0338, -0.0544],\n",
       "                      [-0.0176, -0.0312,  0.0211,  ...,  0.0316,  0.0148,  0.0196],\n",
       "                      [ 0.0217,  0.0252, -0.0286,  ..., -0.0200, -0.0046, -0.0727],\n",
       "                      ...,\n",
       "                      [ 0.0458,  0.0269,  0.0305,  ...,  0.0067, -0.0749,  0.0093],\n",
       "                      [ 0.0028, -0.0258,  0.0405,  ..., -0.0010,  0.0127,  0.0176],\n",
       "                      [ 0.0030, -0.0177,  0.0435,  ...,  0.0021,  0.0059,  0.0307]])),\n",
       "             ('transformer.h.23.self_attention.query_key_value.bias',\n",
       "              tensor([-9.4910e-02, -7.8186e-02,  1.8079e-01,  5.6335e-02, -3.0289e-02,\n",
       "                      -3.8391e-02, -3.1299e-01, -7.6294e-02, -6.8604e-02,  1.9812e-01,\n",
       "                       6.2195e-02, -8.8745e-02, -1.6455e-01,  1.0992e-01, -1.6724e-02,\n",
       "                      -6.5796e-02, -3.6835e-02,  4.0466e-02, -1.7120e-02, -7.4463e-03,\n",
       "                       9.3506e-02, -3.1616e-01,  5.5511e-02,  1.7859e-01, -3.5815e-01,\n",
       "                      -2.8613e-01,  8.4351e-02, -6.9092e-02,  3.2275e-01,  1.2047e-02,\n",
       "                      -1.8909e-01, -1.7310e-01,  2.3303e-01, -2.7759e-01,  2.4585e-01,\n",
       "                      -1.7932e-01,  4.6806e-03,  4.6417e-02,  1.4807e-01,  8.4839e-03,\n",
       "                      -1.2978e-02,  4.4556e-02, -2.3743e-01, -3.9673e-02,  4.8645e-02,\n",
       "                      -3.2153e-01,  3.4692e-01,  1.8784e-02, -1.2585e-01,  2.0981e-03,\n",
       "                      -1.1322e-01,  8.5022e-02,  7.3853e-02,  2.3148e-02, -2.4872e-02,\n",
       "                       6.3721e-02,  9.3872e-02, -2.1286e-02, -2.4731e-01,  1.7957e-01,\n",
       "                      -2.0096e-02, -3.2867e-02,  1.3928e-01,  1.9922e-01,  1.8173e-02,\n",
       "                      -1.4778e-02, -7.3318e-03, -3.0384e-03,  6.1340e-03, -4.7516e-02,\n",
       "                       1.0187e-01,  2.3651e-02, -2.2995e-02, -1.4366e-02, -2.5284e-02,\n",
       "                       1.1314e-02,  6.2103e-02,  1.8082e-02,  5.4321e-02, -2.1667e-02,\n",
       "                       2.4704e-02,  1.2947e-02,  5.2032e-02,  2.0905e-02, -9.0942e-02,\n",
       "                       2.2720e-02,  6.0425e-03, -7.2021e-02, -5.8228e-02,  3.8239e-02,\n",
       "                      -3.4912e-02, -2.6428e-02,  4.6196e-03,  1.3237e-02,  1.2589e-02,\n",
       "                      -1.0574e-02, -4.8370e-02,  2.8320e-02, -3.7476e-02, -1.0920e-03,\n",
       "                       3.8513e-02, -8.0933e-02,  1.2352e-02,  2.8427e-02,  2.9861e-02,\n",
       "                       2.3788e-02,  4.7517e-04, -1.6312e-02,  2.7054e-02,  3.2837e-02,\n",
       "                       3.8666e-02, -4.3564e-03, -1.5434e-02,  5.0873e-02, -2.3102e-02,\n",
       "                      -3.4451e-04, -7.6843e-02, -3.3966e-02,  7.4951e-02, -5.2834e-03,\n",
       "                      -2.3079e-04,  1.1452e-02,  3.7903e-02,  4.8279e-02, -6.8604e-02,\n",
       "                       6.1523e-02, -2.7222e-02,  2.2247e-02,  4.5679e-01,  4.6967e-02,\n",
       "                       3.8623e-01,  2.8125e-01,  4.5654e-01, -1.9170e+00,  8.9172e-02,\n",
       "                       5.4688e-01,  5.0568e-02,  3.9001e-02, -1.0475e-02, -2.0740e-01,\n",
       "                       5.0049e-01, -1.4453e-01, -2.7588e-01,  8.9355e-01, -6.9043e-01,\n",
       "                       8.1787e-01, -1.3867e-01, -1.9446e-01,  9.1919e-02,  2.1619e-01,\n",
       "                      -4.0820e-01,  3.4595e-01,  6.9847e-03,  2.7863e-02,  7.8516e-01,\n",
       "                      -4.5557e-01,  1.0371e+00,  1.2671e-01, -4.2419e-02,  4.1650e-01,\n",
       "                       5.2441e-01, -3.5083e-01,  6.9531e-01, -1.7078e-01,  5.6030e-02,\n",
       "                      -5.6152e-03,  1.2085e-01,  2.3169e-01, -2.4170e-01, -3.4106e-01,\n",
       "                       7.2461e-01,  9.4385e-01,  7.1411e-02, -5.0507e-02, -2.4866e-01,\n",
       "                       3.1592e-01, -6.9775e-01, -3.1079e-01,  1.4624e-01, -6.5576e-01,\n",
       "                      -5.1123e-01, -3.6572e-01, -2.9639e-01,  2.6587e-01,  5.8350e-01,\n",
       "                      -4.8389e-01, -4.0454e-01,  7.9651e-02, -1.5000e+00, -5.1025e-01,\n",
       "                       7.7197e-01, -1.2927e-01, -9.3506e-01, -5.6592e-01,  4.4824e-01,\n",
       "                       2.8296e-01, -1.2158e-01,  7.2656e-01, -8.0029e-01, -2.6562e-01,\n",
       "                       4.8901e-01, -2.1313e-01,  1.0175e-01,  7.5342e-01, -3.3894e-03,\n",
       "                       1.6562e+00,  9.7119e-01,  3.1152e-01, -5.0879e-01,  1.6406e-01,\n",
       "                      -2.9517e-01, -3.9453e-01,  2.7881e-01,  7.8369e-01, -3.7354e-01,\n",
       "                      -5.5615e-01, -1.0712e-01, -7.3853e-02, -9.7412e-01,  6.4844e-01,\n",
       "                       2.6001e-01,  4.2993e-01, -1.0068e+00,  1.7410e-02,  5.4248e-01,\n",
       "                       1.5784e-01, -1.0553e-01,  2.3303e-01, -9.6191e-02,  4.5685e-02,\n",
       "                       4.3018e-01,  5.4736e-01,  1.5698e-01, -4.6338e-01, -2.8760e-01,\n",
       "                       2.1143e-01,  4.5752e-01,  1.5222e-01, -5.2197e-01,  4.4360e-01,\n",
       "                       3.4106e-01,  1.5515e-01,  1.4404e-01, -3.9246e-02, -2.5195e-01,\n",
       "                       5.5664e-01,  4.7778e-01, -1.9861e-01,  3.1274e-01,  3.2788e-01,\n",
       "                       2.2375e-01,  8.7509e-03,  3.0981e-01,  4.4971e-01,  1.5264e+00,\n",
       "                      -3.2349e-01, -2.5272e-03,  4.6448e-02, -3.1769e-02,  7.9285e-02,\n",
       "                      -1.5732e-02, -1.2054e-02,  1.1584e-01, -1.5112e-01,  1.0785e-01,\n",
       "                      -5.4840e-02,  5.6458e-02,  3.3966e-02, -5.3825e-03,  7.3181e-02,\n",
       "                       1.0730e-01, -8.7158e-02,  6.8169e-03, -2.8931e-02, -3.1662e-03,\n",
       "                       5.9326e-02,  1.5762e-02, -1.5745e-03, -1.9455e-03, -2.3788e-02,\n",
       "                       4.4952e-02, -6.5735e-02,  3.0960e-02,  3.3203e-02,  4.3976e-02,\n",
       "                      -2.9816e-02,  5.7587e-02, -2.1229e-03,  5.5450e-02, -3.7956e-03,\n",
       "                      -2.9297e-01,  1.4130e-02,  5.8784e-03, -1.0876e-01,  2.5726e-02,\n",
       "                      -6.9923e-03, -2.3331e-02, -4.3945e-02, -3.7498e-03,  6.3293e-02,\n",
       "                      -2.0737e-02,  1.3145e-02, -3.3417e-02, -1.4702e-02, -1.0199e-01,\n",
       "                       6.0486e-02, -2.5055e-02, -3.3661e-02,  2.1744e-03,  3.8483e-02,\n",
       "                       1.9150e-02,  4.6295e-02, -3.2166e-02, -7.3608e-02,  8.8562e-02,\n",
       "                      -1.4221e-02,  2.2476e-02,  8.8196e-02, -3.4485e-02, -1.3962e-02,\n",
       "                       2.7344e-01, -4.1797e-01,  9.8083e-02,  7.2412e-01, -2.2205e-01,\n",
       "                      -8.4778e-02,  5.6641e-02,  5.5811e-01, -2.6440e-01,  2.0569e-01,\n",
       "                       6.7920e-01,  4.2285e-01,  2.4854e-01,  1.4626e-02, -5.1611e-01,\n",
       "                       4.2041e-01, -1.3969e-02, -3.9429e-02, -4.5483e-01, -2.7979e-01,\n",
       "                       3.9795e-01, -1.8237e-01, -2.3059e-01,  6.8408e-01, -1.0065e-01,\n",
       "                       2.5684e-01, -1.1749e-01, -8.0859e-01, -5.5420e-01,  7.5378e-03,\n",
       "                      -1.3496e-02, -3.9600e-01, -3.9673e-01,  6.5842e-03,  6.9809e-03,\n",
       "                       1.5002e-01, -2.7905e-01, -1.1804e-01, -5.7666e-01, -5.0995e-02,\n",
       "                      -3.9966e-01, -3.3057e-01, -3.1445e-01, -3.9502e-01,  2.7100e-01,\n",
       "                      -2.7026e-01,  2.8125e-01, -1.7944e-01,  1.8884e-01, -1.3562e-01,\n",
       "                      -1.5137e-01,  3.7903e-02, -7.6721e-02,  1.8506e-01,  2.1350e-01,\n",
       "                      -3.6450e-01,  4.2114e-01,  2.0496e-01, -3.0884e-01,  2.1045e-01,\n",
       "                       1.2140e-01,  9.4482e-02,  8.9539e-02,  5.7281e-02,  2.2285e+00,\n",
       "                      -9.4116e-02,  2.3926e+00, -8.2178e-01,  3.0884e-01,  1.6769e-02,\n",
       "                      -1.7354e+00,  1.3623e+00, -9.8730e-01, -1.7944e-01, -2.3164e+00,\n",
       "                       2.1887e-01,  6.1475e-01,  2.4473e+00,  4.3896e-01, -2.1367e+00,\n",
       "                       1.7305e+00, -2.6709e-01, -3.6914e-01,  1.0596e+00,  1.6025e+00,\n",
       "                      -2.4551e+00,  1.7471e+00, -5.0098e-01,  2.0044e-01,  1.9092e+00,\n",
       "                       1.7725e+00, -3.0352e+00,  1.2158e+00, -1.1445e+00,  6.0107e-01,\n",
       "                       1.4668e+00, -2.4590e+00,  1.1650e+00, -1.4717e+00,  1.9814e+00,\n",
       "                       1.8689e-01, -1.9639e+00,  1.0752e+00,  1.7871e+00,  3.2251e-01,\n",
       "                      -3.9429e-01,  2.2227e+00, -1.1060e-01,  7.2705e-01, -3.5176e+00,\n",
       "                       2.1172e+00, -6.1670e-01,  4.3320e+00, -8.4961e-01, -4.0894e-01,\n",
       "                       2.6270e+00,  3.6504e+00,  6.0303e-01,  1.2061e+00, -2.8711e+00,\n",
       "                       7.9053e-01,  5.6885e-02,  7.5977e-01,  1.2103e-01,  2.3711e+00,\n",
       "                       2.1973e+00, -6.8398e+00,  2.4622e-01,  3.5430e+00, -6.7969e-01,\n",
       "                      -2.1719e+00, -3.7891e+00, -2.0801e+00,  1.3027e+00,  4.8340e-01,\n",
       "                      -2.3669e-01,  7.0361e-01, -8.6133e-01,  3.2285e+00, -1.7598e+00,\n",
       "                       2.2480e+00, -2.3340e+00, -4.7773e+00,  2.7188e+00,  3.5098e+00,\n",
       "                       2.3281e+00, -2.2034e-01,  1.5586e+00,  1.7559e+00,  2.9160e+00,\n",
       "                       1.0254e+00,  8.0312e+00,  2.0059e+00, -2.9453e+00,  3.3906e+00,\n",
       "                       1.2617e+00,  2.5820e+00, -1.4014e+00,  7.7832e-01,  3.2285e+00,\n",
       "                      -4.0723e-01, -3.9600e-01,  4.7461e+00, -7.9395e-01, -5.8105e-01,\n",
       "                      -1.2451e+00,  1.0732e+00,  1.5391e+00, -1.5312e+00, -3.3086e+00,\n",
       "                      -4.8906e+00,  3.2637e+00,  1.3301e+00,  2.5762e+00, -3.8340e+00,\n",
       "                      -4.5234e+00, -2.7891e+00, -6.6016e+00, -5.5430e+00,  2.6001e-01,\n",
       "                       7.6709e-01,  9.4727e-01, -1.7358e-01, -4.6797e+00, -3.5273e+00,\n",
       "                      -3.1152e+00,  1.6931e-01, -1.0078e+01,  1.6248e-01,  8.5327e-02,\n",
       "                      -1.7607e+00,  1.4111e+00, -1.0077e-01, -5.6104e-01, -8.6243e-02,\n",
       "                       4.5288e-01,  4.4092e-01,  5.4395e-01,  8.9307e-01,  3.9404e-01,\n",
       "                       2.4170e-01,  6.2646e-01,  5.8398e-01,  6.0449e-01, -8.4045e-02,\n",
       "                       1.6370e-01, -4.2188e-01, -7.8125e-01, -3.4106e-01, -2.4976e-01,\n",
       "                       7.4707e-01,  1.4233e-01,  2.5269e-01,  3.1714e-01, -1.5662e-01,\n",
       "                      -2.1240e-02,  9.7412e-01, -8.0017e-02,  3.4424e-02,  2.4319e-03,\n",
       "                       1.8723e-02,  3.2861e-01, -9.6863e-02, -1.6191e+00,  6.5771e-01,\n",
       "                      -5.7007e-02, -1.6614e-01,  3.4717e-01,  1.0010e+00, -4.3750e-01,\n",
       "                       8.1482e-02,  1.3672e-01,  2.8760e-01, -2.1558e-01, -7.6294e-02,\n",
       "                       5.1025e-01,  2.0923e-01,  2.0035e-02, -4.4702e-01, -5.6183e-02,\n",
       "                      -1.7761e-01, -2.3682e-01, -2.1332e-02, -2.4023e-01,  6.7871e-02,\n",
       "                      -4.1968e-01,  9.6094e-01,  5.3673e-03,  4.2725e-01, -2.0667e-01,\n",
       "                      -2.3352e-01, -3.7689e-02,  1.2805e-01,  5.9131e-01, -1.3633e+00,\n",
       "                       8.6670e-01, -2.1460e-01, -8.8477e-01,  2.0449e+00, -7.8857e-01,\n",
       "                       4.4336e-01,  3.5205e-01,  8.2109e+00, -6.5039e-01, -2.7480e+00,\n",
       "                      -4.3701e-02, -2.6172e+00,  1.9805e+00,  1.8262e-01,  3.8574e+00,\n",
       "                       4.3711e+00, -5.6641e+00,  1.0537e+00, -2.4785e+00, -3.8535e+00,\n",
       "                      -6.6172e+00, -2.0332e+00, -1.4648e+00,  9.0332e-01,  3.6895e+00,\n",
       "                      -4.6523e+00,  3.7734e+00, -7.9736e-01, -4.7305e+00, -4.4922e+00,\n",
       "                       7.5625e+00, -3.1016e+00, -4.7656e+00, -4.4214e-01,  1.8604e+00,\n",
       "                      -3.3496e+00,  9.2432e-01, -4.0977e+00, -5.3242e+00, -1.3984e+00,\n",
       "                      -1.0727e+01,  4.5664e+00, -3.7793e+00,  2.6016e+00,  6.9375e+00,\n",
       "                      -1.8125e+00, -3.4160e+00, -2.0020e+00, -6.6367e+00,  5.5859e-01,\n",
       "                      -1.1719e+00,  2.5762e+00,  3.1523e+00, -4.6055e+00, -5.7344e+00,\n",
       "                      -1.2609e+01,  4.2148e+00,  3.5195e+00,  1.3938e+01, -5.3125e+00,\n",
       "                       1.9385e+00, -4.8242e+00, -3.2012e+00,  1.5173e-01, -5.0039e+00,\n",
       "                      -3.1891e-02,  2.3127e-05,  3.7598e-02,  3.8025e-02, -2.9480e-02,\n",
       "                      -3.4515e-02, -6.5369e-02,  9.8572e-03,  4.9347e-02, -8.9417e-02,\n",
       "                      -1.2955e-02, -1.4046e-02,  5.2856e-02,  6.9809e-03,  1.2054e-02,\n",
       "                       5.0385e-02,  7.6408e-03, -2.0386e-02, -1.5442e-02, -1.6052e-02,\n",
       "                      -5.7098e-02, -1.3428e-02, -5.0476e-02,  2.1420e-03,  1.0651e-01,\n",
       "                       4.9629e-03,  1.1896e-01,  7.3792e-02, -4.7836e-03,  6.4880e-02,\n",
       "                      -4.0741e-02,  3.9917e-02, -4.3716e-03, -2.7924e-02, -9.4543e-02,\n",
       "                       7.3547e-02, -8.1665e-02,  8.0444e-02,  4.2419e-02, -4.5746e-02,\n",
       "                      -1.8895e-04, -2.0477e-02, -9.9258e-03,  2.3613e-03,  4.5593e-02,\n",
       "                       9.4482e-02, -6.0043e-03,  2.0493e-02, -6.3904e-02, -5.8517e-03,\n",
       "                      -3.4515e-02,  2.7435e-02,  5.4138e-02, -2.8137e-02,  2.0721e-02,\n",
       "                      -5.5456e-04, -6.1157e-02,  4.7852e-02, -7.2975e-03,  4.0100e-02,\n",
       "                      -1.1650e-02,  2.8107e-02,  9.6130e-04, -2.0706e-02,  6.0181e-02,\n",
       "                      -2.0050e-02,  5.6335e-02, -4.0436e-02,  5.5328e-02,  1.1269e-02,\n",
       "                      -6.1127e-02, -1.0315e-02, -5.1239e-02,  2.2430e-02, -4.0436e-02,\n",
       "                       7.1167e-02,  6.3232e-02,  5.1308e-03,  1.1307e-02,  4.0466e-02,\n",
       "                       4.5853e-03,  9.5520e-03,  2.7557e-02, -5.6976e-02,  2.0828e-03,\n",
       "                      -6.8787e-02,  4.7394e-02,  3.4363e-02, -1.0071e-02,  3.9612e-02,\n",
       "                       2.9175e-02,  6.1798e-02,  4.7821e-02, -9.6664e-03, -1.4740e-02,\n",
       "                       1.0742e-01,  3.3142e-02,  2.9449e-02, -7.2205e-02, -2.3834e-02,\n",
       "                       1.1322e-01,  7.1335e-03, -7.1228e-02,  2.8778e-02, -2.8412e-02,\n",
       "                      -2.1576e-02, -8.2397e-02, -1.3147e-01, -6.1432e-02, -7.4341e-02,\n",
       "                      -1.7128e-03,  5.3802e-02,  1.0309e-01, -2.7222e-02,  6.8542e-02,\n",
       "                       3.1376e-03, -7.8857e-02, -6.0791e-02,  1.6528e-01, -1.3977e-02,\n",
       "                       1.6739e-02, -2.0065e-02,  3.5248e-02, -1.9653e-02,  3.0334e-02,\n",
       "                       7.0740e-02,  8.6304e-02, -4.2786e-02])),\n",
       "             ('transformer.h.23.self_attention.dense.weight',\n",
       "              tensor([[-0.0015,  0.0010,  0.0075,  ...,  0.0043,  0.0023,  0.0131],\n",
       "                      [ 0.0024,  0.0084,  0.0038,  ...,  0.0002, -0.0022, -0.0064],\n",
       "                      [ 0.0070,  0.0162, -0.0069,  ...,  0.0011,  0.0028,  0.0015],\n",
       "                      ...,\n",
       "                      [ 0.0152,  0.0050, -0.0223,  ..., -0.0172,  0.0062, -0.0504],\n",
       "                      [-0.0113, -0.0056,  0.0035,  ...,  0.0036,  0.0005, -0.0028],\n",
       "                      [-0.0100, -0.0102,  0.0117,  ..., -0.0027, -0.0075, -0.0026]])),\n",
       "             ('transformer.h.23.self_attention.dense.bias',\n",
       "              tensor([-0.0264,  0.0090,  0.0239,  ..., -1.1992,  0.0221,  0.0067])),\n",
       "             ('transformer.h.23.post_attention_layernorm.weight',\n",
       "              tensor([3.4453, 3.9180, 3.8418,  ..., 1.3809, 3.7598, 3.4492])),\n",
       "             ('transformer.h.23.post_attention_layernorm.bias',\n",
       "              tensor([-0.0685,  0.1493, -0.2021,  ...,  0.3176, -0.0578,  0.0466])),\n",
       "             ('transformer.h.23.mlp.dense_h_to_4h.weight',\n",
       "              tensor([[ 0.0170, -0.0080,  0.0056,  ..., -0.0069,  0.0074,  0.0208],\n",
       "                      [-0.0096, -0.0146,  0.0037,  ..., -0.0011,  0.0072,  0.0045],\n",
       "                      [-0.0122,  0.0170,  0.0051,  ..., -0.0069, -0.0049, -0.0156],\n",
       "                      ...,\n",
       "                      [-0.0039, -0.0068,  0.0156,  ...,  0.0001,  0.0147,  0.0464],\n",
       "                      [ 0.0138,  0.0221,  0.0588,  ..., -0.0064,  0.0352, -0.0274],\n",
       "                      [-0.0091,  0.0157, -0.0146,  ...,  0.0012,  0.0232, -0.0090]])),\n",
       "             ('transformer.h.23.mlp.dense_h_to_4h.bias',\n",
       "              tensor([-0.2964, -0.4802,  0.0624, -0.1650, -0.1820, -0.3689, -0.1547, -0.2698,\n",
       "                      -0.2517,  0.1613, -0.1098, -0.2052, -0.3762,  0.3076, -0.1521, -0.1462,\n",
       "                      -0.4480, -0.3206, -0.0695, -0.1926, -0.2263, -0.2297,  0.0147, -0.0230,\n",
       "                       0.0882, -0.2957, -0.2460, -0.2491, -0.1541, -0.2153, -0.3423, -0.3977,\n",
       "                      -0.3816,  0.0925,  0.1151, -0.6191, -0.0598, -0.4829, -0.0304, -0.1953,\n",
       "                      -0.3169,  0.2871, -0.3611, -0.0791, -0.3496,  0.0224, -0.1541, -0.2957,\n",
       "                       0.1049, -0.1956, -0.2428,  0.0829, -0.1976, -0.1224,  0.1074, -0.5078,\n",
       "                      -0.3149, -0.5532, -0.2157, -0.0143,  0.2145, -0.2295,  0.6040, -0.2108,\n",
       "                      -0.0381, -0.3088, -0.0370,  0.2485, -0.1146,  0.0984, -0.1670, -0.3970,\n",
       "                      -0.3293, -0.0840,  0.1293, -0.5059,  0.1375, -0.1539, -0.2812, -0.5684,\n",
       "                       0.1981, -0.2367, -0.1641, -0.2026, -0.0759, -0.2893, -0.4128, -0.0988,\n",
       "                      -0.4131, -0.6113, -0.1415, -0.1945,  0.4543, -0.3818, -0.3130,  0.1964,\n",
       "                      -0.0150, -0.4702, -0.1860, -0.1148, -0.2054, -0.2764, -0.1004, -0.2810,\n",
       "                      -0.2308, -0.7354,  0.4629, -0.0189, -0.2961,  0.2712, -0.4292, -0.0726,\n",
       "                      -0.2075,  0.2303, -0.1958,  0.2001, -0.1931,  0.1533, -0.2842, -0.3274,\n",
       "                      -0.0933,  0.1482,  0.0715, -0.1943, -0.6553, -0.0453, -0.1289, -0.1543,\n",
       "                      -0.4133,  0.1587, -0.2910, -0.5063, -0.1262, -0.4036, -0.2340, -0.2170,\n",
       "                       0.0123, -0.2722, -0.5093, -0.0747, -1.3682, -0.1052, -0.1428,  0.2935,\n",
       "                      -0.7446,  0.2705, -0.3176, -0.2505, -0.8213, -0.5186, -0.0801, -0.3325,\n",
       "                      -0.0017, -0.2394, -0.3357, -0.4041,  0.3420, -0.6792, -0.1990, -0.3040,\n",
       "                      -0.2252, -0.5566, -0.0969, -0.7529, -0.2322, -0.1266, -0.4922, -0.3704,\n",
       "                      -0.4666, -0.2002,  0.1609, -0.3035, -0.2952,  0.0238, -0.0237, -0.3735,\n",
       "                      -0.2861, -0.3359, -0.5811,  0.1575,  0.2578,  0.3396, -0.2859, -0.1831,\n",
       "                      -0.1770, -0.2502, -0.2605, -0.3457, -0.6216,  0.1538, -0.2937, -0.2561,\n",
       "                      -0.0380, -0.4297, -0.0822, -0.5391, -0.2871, -0.6660, -0.3643, -0.1649,\n",
       "                       0.2642, -0.3115, -0.3555, -0.2664, -0.0195, -0.1216,  0.3149, -0.8018,\n",
       "                      -0.2917, -0.4512, -0.3074, -0.3154, -0.3467, -0.1182, -0.1005,  0.0108,\n",
       "                      -0.2744, -0.2654, -0.1216, -0.1304, -0.0916, -0.4871, -0.4277,  0.4180,\n",
       "                       0.2156, -0.3630, -0.3499, -0.4312, -0.3252, -0.1235, -0.3208, -0.3428,\n",
       "                      -0.0121, -0.1681, -0.0216, -0.0210, -0.1528, -0.3621, -0.2561, -0.3572,\n",
       "                       0.2140, -0.0345, -0.1136, -0.0265,  0.6377, -0.2325, -0.1041, -0.1746,\n",
       "                      -0.1122,  0.3013, -0.4844, -0.4639, -0.7764, -0.0771, -0.5601, -0.2025,\n",
       "                      -0.0516,  0.0662, -0.2493, -0.2896, -0.2620, -0.6948, -0.0416,  0.1753,\n",
       "                      -0.3877, -0.4578,  0.0935,  0.2620,  0.2319, -0.2668, -0.2430, -0.5098,\n",
       "                      -0.0058, -0.1873, -0.2771,  0.5029, -0.0237, -0.2175, -0.2469,  0.0676,\n",
       "                      -0.1595, -0.3015, -0.5078,  0.0351, -0.5713, -0.4272, -0.1064, -0.1414,\n",
       "                      -0.4985, -0.4363,  0.0156, -0.2145, -0.2225,  0.1169,  0.1594, -0.3818,\n",
       "                       0.1218, -0.2224,  0.0659, -0.3406,  0.6802, -0.1792, -0.3242, -0.1808,\n",
       "                      -0.1842, -0.3379, -0.3269, -0.1354, -0.1951, -0.1349, -0.3113,  0.0397,\n",
       "                       0.3098, -0.2136, -0.0656, -0.2292, -0.2644,  0.0159, -0.1154, -0.3840,\n",
       "                      -0.4587, -0.3499, -0.4734, -0.4131, -0.2791, -0.0363,  0.2094, -0.3345,\n",
       "                      -0.3997, -0.6699, -0.2544, -0.6372,  0.4338,  0.2571,  0.2305,  0.0746,\n",
       "                      -0.4858, -0.4060, -0.2103, -0.2355,  0.4612, -0.4949, -0.0534,  0.0818,\n",
       "                      -0.3274, -0.1660, -0.2903, -0.0638, -0.1660, -0.0272,  0.2017,  0.2307,\n",
       "                      -0.0782, -0.2549, -0.0075, -0.5742, -0.5571, -0.3394, -0.2268, -0.2484,\n",
       "                      -0.5425, -0.0520, -0.2106,  0.0251, -0.7241, -0.1379, -0.1069, -0.0715,\n",
       "                      -0.2576, -0.3252, -0.1489, -0.4836, -0.2578, -0.5547,  0.0185, -0.2427,\n",
       "                      -0.1373, -0.0113, -0.3569, -0.3547, -0.3333, -0.1772, -0.2925, -0.0317,\n",
       "                      -0.2588, -0.2827, -0.0602,  0.1710, -0.3306, -0.2190, -0.2764, -0.1558,\n",
       "                      -0.5952,  0.2108,  0.0812, -0.1930, -0.2808, -0.0412,  0.1882, -0.3945,\n",
       "                       0.3921, -0.3701, -0.2250, -0.3535, -0.3403, -0.0501,  0.1122, -0.1470,\n",
       "                      -0.3906, -0.1161, -0.1958, -0.7632, -0.1320,  0.2179, -0.3403,  0.0861,\n",
       "                      -0.6787, -0.1262,  0.2410, -0.1610, -0.3250, -0.0605, -0.0482, -0.3740,\n",
       "                      -0.3914, -0.2438, -0.5439, -0.4568, -0.4216, -0.1312, -0.4077, -0.4785,\n",
       "                      -0.4812, -0.1219, -0.4487, -0.8257,  0.2036, -0.1890, -0.6772, -0.0645,\n",
       "                       0.3203, -0.1392, -0.3511, -0.2952, -0.3064, -0.2617, -0.0547, -0.4839,\n",
       "                      -0.2283, -0.0950, -0.1251, -0.0543,  0.1401, -0.0834, -0.1335,  0.0533,\n",
       "                      -0.2452, -0.1366, -0.5322, -0.2576,  0.0588, -0.0353, -0.4956,  0.5283,\n",
       "                      -0.3567, -0.3860, -0.0541,  0.3098, -0.7217, -0.7227, -0.6533, -0.4712,\n",
       "                      -0.1411,  0.0256, -0.2703,  0.4817, -0.3503, -0.3120, -0.2927, -0.2959,\n",
       "                      -0.3572, -0.5322, -0.2097, -0.4434, -0.5249, -0.3352, -0.4202, -0.2688,\n",
       "                      -0.2373, -0.0738,  0.0964, -0.0823, -0.3140, -0.3569, -0.1234, -0.6719,\n",
       "                      -0.3960, -0.1187, -0.5625, -0.0404, -0.3892, -0.7886, -0.2289, -0.2854,\n",
       "                       0.1180, -0.3979, -0.3208, -0.4070, -0.3369, -0.3096, -0.0220, -0.1348,\n",
       "                      -0.2698, -0.1515, -0.2974, -0.1906,  0.1098,  0.0524, -0.6270, -0.1426,\n",
       "                      -0.2725, -0.0692, -0.6841, -0.3250,  0.0504, -0.2507, -0.1163, -0.1209,\n",
       "                      -0.3513, -0.1975, -0.3677, -0.0976, -0.1179, -0.1674, -0.3357, -0.2269,\n",
       "                      -0.4692,  0.0734, -0.4111, -0.4014, -0.1685, -0.4429,  0.0898, -0.4006,\n",
       "                      -0.2286, -0.2238, -0.1226, -0.1504, -0.1054, -0.3918,  0.5952, -0.3845,\n",
       "                      -0.1650,  0.1858, -0.1920, -0.0662, -0.3665, -0.3606,  0.1371,  0.3767,\n",
       "                      -0.2964, -0.4851,  0.0731,  0.0875, -0.1046, -0.5742, -0.2241, -0.0908,\n",
       "                       0.0282, -0.7070, -0.6245,  0.0575, -0.2502,  0.1372,  0.2690, -0.0963,\n",
       "                       0.0103, -0.3977,  0.2175, -0.3132, -0.2795, -0.1220, -0.5981, -0.3054,\n",
       "                      -0.1267, -0.2632, -0.1959, -0.2515, -0.0964, -0.1447, -0.0645, -0.3247,\n",
       "                      -0.2393,  0.1614, -0.2903, -0.2527,  0.0976, -0.3052, -0.2991, -0.4807,\n",
       "                      -0.3486,  0.0288,  0.0735, -0.4207, -0.4502, -0.1672, -0.1863, -0.2502,\n",
       "                      -0.2773, -0.2988, -0.4011, -0.2896, -0.2573,  0.1399, -0.1924, -0.2612,\n",
       "                      -0.3450, -0.1119,  0.1274,  0.0276, -0.3872, -0.5737, -0.4092, -0.7178,\n",
       "                      -0.0370, -0.2390, -0.1389, -0.2617, -0.4500, -0.2133, -0.4739,  0.5093,\n",
       "                      -0.5977,  0.2448,  0.1274, -0.5137, -0.1304, -0.2078, -0.7310, -0.6948,\n",
       "                      -0.4929, -0.2404, -0.1555, -0.2395,  0.1140, -0.5312, -0.3052, -0.4485,\n",
       "                      -0.2620, -0.2625, -0.4380, -0.2358, -0.0805,  0.0065, -0.3728, -0.3037,\n",
       "                      -0.2832, -0.2101, -0.1392, -0.2920, -0.4402, -0.2446, -0.5547, -0.1620,\n",
       "                      -0.1493, -0.3496, -0.1102,  0.1007, -0.2881, -0.8247, -0.0129,  0.3936,\n",
       "                      -0.1495, -0.2034, -0.3303, -0.4109, -0.5669, -0.2900, -0.2347, -0.2388,\n",
       "                      -0.3276,  0.2476, -0.6875, -0.4102, -0.3682, -0.2266, -0.3604, -0.1819,\n",
       "                      -0.2305, -0.3728, -0.0649,  0.2024, -0.1183, -0.1249, -0.2382, -0.0537,\n",
       "                       0.1361, -0.3772, -0.2032,  0.0055, -0.1670, -0.3479, -0.1099, -0.4021,\n",
       "                      -0.0908, -0.6152,  0.1975, -0.6299,  0.2627, -0.1525,  0.3696, -0.1969,\n",
       "                      -0.3606,  0.1375, -0.1353, -0.3994,  0.4839, -0.1783,  0.3259,  0.2438,\n",
       "                      -0.2561, -0.5903,  0.0882,  0.4243, -0.1627, -0.1466, -0.2305, -0.2283,\n",
       "                      -0.6118, -0.2961,  0.1962,  0.2332, -0.6650,  0.0673, -0.0731, -0.3149,\n",
       "                       0.2583, -0.1729, -0.1437,  0.2333,  0.0413, -0.4255, -0.3430, -0.5083,\n",
       "                      -0.1036, -0.1343, -0.2664, -0.4507, -0.4207,  0.0455, -0.2327, -0.2441,\n",
       "                      -0.0571, -0.1202, -0.2151, -0.1847, -0.2546, -0.3145, -0.1832, -0.1148,\n",
       "                      -0.3120, -0.2903, -0.0421, -0.2399, -0.3083, -0.2057, -0.3594, -0.0972,\n",
       "                      -0.2188, -0.1042, -0.6860, -0.1616, -0.1150, -0.0930, -0.0697, -0.6836,\n",
       "                      -0.5620, -0.2590,  0.1971, -0.6074, -0.2766])),\n",
       "             ('transformer.h.23.mlp.dense_4h_to_h.weight',\n",
       "              tensor([[-7.9346e-03,  3.4904e-03,  6.4125e-03,  ...,  1.6464e-02,\n",
       "                       -1.8051e-02,  1.6144e-02],\n",
       "                      [ 6.2218e-03, -1.0033e-02,  2.0737e-02,  ..., -4.1580e-03,\n",
       "                       -1.7424e-03, -3.7048e-02],\n",
       "                      [ 3.8548e-03,  3.3630e-02, -2.0828e-02,  ...,  3.7251e-03,\n",
       "                       -4.2358e-02, -1.4145e-02],\n",
       "                      ...,\n",
       "                      [-3.9005e-03, -5.9662e-03, -4.0703e-03,  ..., -9.4891e-05,\n",
       "                       -4.8256e-03, -7.5302e-03],\n",
       "                      [ 2.2945e-03, -8.2550e-03,  1.6891e-02,  ..., -1.3664e-02,\n",
       "                       -4.9683e-02, -2.0933e-04],\n",
       "                      [-4.8859e-02, -1.1185e-02,  1.2993e-02,  ..., -5.7098e-02,\n",
       "                        2.2354e-02, -1.0681e-02]])),\n",
       "             ('transformer.h.23.mlp.dense_4h_to_h.bias',\n",
       "              tensor([-0.1279, -0.1272, -0.0613,  ...,  0.2291,  0.0349, -0.0659])),\n",
       "             ('transformer.ln_f.weight',\n",
       "              tensor([17.8594, 17.5000, 16.6875,  ..., 88.5625, 18.0000, 17.5156])),\n",
       "             ('transformer.ln_f.bias',\n",
       "              tensor([-1.9470e-02, -2.0874e-02,  3.2135e-02,  ..., -6.7000e+01,\n",
       "                      -1.0364e-01, -1.4490e-01])),\n",
       "             ('lm_head.weight',\n",
       "              tensor([[-0.0099, -0.0048, -0.0111,  ..., -0.0426,  0.0099,  0.0212],\n",
       "                      [ 0.0048, -0.0127,  0.0138,  ..., -0.0448,  0.0003, -0.0120],\n",
       "                      [ 0.0065,  0.0239,  0.0050,  ..., -0.0431, -0.0067,  0.0137],\n",
       "                      ...,\n",
       "                      [-0.0028, -0.0038, -0.0012,  ..., -0.0252,  0.0013,  0.0012],\n",
       "                      [-0.0028, -0.0038, -0.0012,  ..., -0.0252,  0.0013,  0.0012],\n",
       "                      [-0.0028, -0.0038, -0.0012,  ..., -0.0252,  0.0013,  0.0012]]))])"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pruned_model_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "fdb4a783-6cd1-44f9-b4e9-a3dd2f5b0156",
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(pruned_model_params, \"pruned_30percent_560m_bloom.pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "9b70a7e7-b902-4ecd-8c82-bb0764f3b9fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "state_dict_shapes = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "7ae74350-2038-435f-b708-e38f0cdee675",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "transformer.word_embeddings.weight torch.Size([250880, 1024])\n",
      "transformer.word_embeddings_layernorm.weight torch.Size([1024])\n",
      "transformer.word_embeddings_layernorm.bias torch.Size([1024])\n",
      "transformer.h.0.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.0.input_layernorm.bias torch.Size([1024])\n",
      "transformer.h.0.self_attention.query_key_value.weight torch.Size([768, 1024])\n",
      "transformer.h.0.self_attention.query_key_value.bias torch.Size([768])\n",
      "transformer.h.0.self_attention.dense.weight torch.Size([1024, 256])\n",
      "transformer.h.0.self_attention.dense.bias torch.Size([1024])\n",
      "transformer.h.0.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.0.post_attention_layernorm.bias torch.Size([1024])\n",
      "transformer.h.0.mlp.dense_h_to_4h.weight torch.Size([1434, 1024])\n",
      "transformer.h.0.mlp.dense_h_to_4h.bias torch.Size([1434])\n",
      "transformer.h.0.mlp.dense_4h_to_h.weight torch.Size([1024, 1434])\n",
      "transformer.h.0.mlp.dense_4h_to_h.bias torch.Size([1024])\n",
      "transformer.h.1.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.1.input_layernorm.bias torch.Size([1024])\n",
      "transformer.h.1.self_attention.query_key_value.weight torch.Size([768, 1024])\n",
      "transformer.h.1.self_attention.query_key_value.bias torch.Size([768])\n",
      "transformer.h.1.self_attention.dense.weight torch.Size([1024, 256])\n",
      "transformer.h.1.self_attention.dense.bias torch.Size([1024])\n",
      "transformer.h.1.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.1.post_attention_layernorm.bias torch.Size([1024])\n",
      "transformer.h.1.mlp.dense_h_to_4h.weight torch.Size([1398, 1024])\n",
      "transformer.h.1.mlp.dense_h_to_4h.bias torch.Size([1398])\n",
      "transformer.h.1.mlp.dense_4h_to_h.weight torch.Size([1024, 1398])\n",
      "transformer.h.1.mlp.dense_4h_to_h.bias torch.Size([1024])\n",
      "transformer.h.2.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.2.input_layernorm.bias torch.Size([1024])\n",
      "transformer.h.2.self_attention.query_key_value.weight torch.Size([768, 1024])\n",
      "transformer.h.2.self_attention.query_key_value.bias torch.Size([768])\n",
      "transformer.h.2.self_attention.dense.weight torch.Size([1024, 256])\n",
      "transformer.h.2.self_attention.dense.bias torch.Size([1024])\n",
      "transformer.h.2.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.2.post_attention_layernorm.bias torch.Size([1024])\n",
      "transformer.h.2.mlp.dense_h_to_4h.weight torch.Size([1612, 1024])\n",
      "transformer.h.2.mlp.dense_h_to_4h.bias torch.Size([1612])\n",
      "transformer.h.2.mlp.dense_4h_to_h.weight torch.Size([1024, 1612])\n",
      "transformer.h.2.mlp.dense_4h_to_h.bias torch.Size([1024])\n",
      "transformer.h.3.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.3.input_layernorm.bias torch.Size([1024])\n",
      "transformer.h.3.self_attention.query_key_value.weight torch.Size([768, 1024])\n",
      "transformer.h.3.self_attention.query_key_value.bias torch.Size([768])\n",
      "transformer.h.3.self_attention.dense.weight torch.Size([1024, 256])\n",
      "transformer.h.3.self_attention.dense.bias torch.Size([1024])\n",
      "transformer.h.3.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.3.post_attention_layernorm.bias torch.Size([1024])\n",
      "transformer.h.3.mlp.dense_h_to_4h.weight torch.Size([1630, 1024])\n",
      "transformer.h.3.mlp.dense_h_to_4h.bias torch.Size([1630])\n",
      "transformer.h.3.mlp.dense_4h_to_h.weight torch.Size([1024, 1630])\n",
      "transformer.h.3.mlp.dense_4h_to_h.bias torch.Size([1024])\n",
      "transformer.h.4.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.4.input_layernorm.bias torch.Size([1024])\n",
      "transformer.h.4.self_attention.query_key_value.weight torch.Size([768, 1024])\n",
      "transformer.h.4.self_attention.query_key_value.bias torch.Size([768])\n",
      "transformer.h.4.self_attention.dense.weight torch.Size([1024, 256])\n",
      "transformer.h.4.self_attention.dense.bias torch.Size([1024])\n",
      "transformer.h.4.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.4.post_attention_layernorm.bias torch.Size([1024])\n",
      "transformer.h.4.mlp.dense_h_to_4h.weight torch.Size([1549, 1024])\n",
      "transformer.h.4.mlp.dense_h_to_4h.bias torch.Size([1549])\n",
      "transformer.h.4.mlp.dense_4h_to_h.weight torch.Size([1024, 1549])\n",
      "transformer.h.4.mlp.dense_4h_to_h.bias torch.Size([1024])\n",
      "transformer.h.5.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.5.input_layernorm.bias torch.Size([1024])\n",
      "transformer.h.5.self_attention.query_key_value.weight torch.Size([768, 1024])\n",
      "transformer.h.5.self_attention.query_key_value.bias torch.Size([768])\n",
      "transformer.h.5.self_attention.dense.weight torch.Size([1024, 256])\n",
      "transformer.h.5.self_attention.dense.bias torch.Size([1024])\n",
      "transformer.h.5.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.5.post_attention_layernorm.bias torch.Size([1024])\n",
      "transformer.h.5.mlp.dense_h_to_4h.weight torch.Size([1440, 1024])\n",
      "transformer.h.5.mlp.dense_h_to_4h.bias torch.Size([1440])\n",
      "transformer.h.5.mlp.dense_4h_to_h.weight torch.Size([1024, 1440])\n",
      "transformer.h.5.mlp.dense_4h_to_h.bias torch.Size([1024])\n",
      "transformer.h.6.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.6.input_layernorm.bias torch.Size([1024])\n",
      "transformer.h.6.self_attention.query_key_value.weight torch.Size([768, 1024])\n",
      "transformer.h.6.self_attention.query_key_value.bias torch.Size([768])\n",
      "transformer.h.6.self_attention.dense.weight torch.Size([1024, 256])\n",
      "transformer.h.6.self_attention.dense.bias torch.Size([1024])\n",
      "transformer.h.6.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.6.post_attention_layernorm.bias torch.Size([1024])\n",
      "transformer.h.6.mlp.dense_h_to_4h.weight torch.Size([1575, 1024])\n",
      "transformer.h.6.mlp.dense_h_to_4h.bias torch.Size([1575])\n",
      "transformer.h.6.mlp.dense_4h_to_h.weight torch.Size([1024, 1575])\n",
      "transformer.h.6.mlp.dense_4h_to_h.bias torch.Size([1024])\n",
      "transformer.h.7.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.7.input_layernorm.bias torch.Size([1024])\n",
      "transformer.h.7.self_attention.query_key_value.weight torch.Size([768, 1024])\n",
      "transformer.h.7.self_attention.query_key_value.bias torch.Size([768])\n",
      "transformer.h.7.self_attention.dense.weight torch.Size([1024, 256])\n",
      "transformer.h.7.self_attention.dense.bias torch.Size([1024])\n",
      "transformer.h.7.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.7.post_attention_layernorm.bias torch.Size([1024])\n",
      "transformer.h.7.mlp.dense_h_to_4h.weight torch.Size([1147, 1024])\n",
      "transformer.h.7.mlp.dense_h_to_4h.bias torch.Size([1147])\n",
      "transformer.h.7.mlp.dense_4h_to_h.weight torch.Size([1024, 1147])\n",
      "transformer.h.7.mlp.dense_4h_to_h.bias torch.Size([1024])\n",
      "transformer.h.8.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.8.input_layernorm.bias torch.Size([1024])\n",
      "transformer.h.8.self_attention.query_key_value.weight torch.Size([768, 1024])\n",
      "transformer.h.8.self_attention.query_key_value.bias torch.Size([768])\n",
      "transformer.h.8.self_attention.dense.weight torch.Size([1024, 256])\n",
      "transformer.h.8.self_attention.dense.bias torch.Size([1024])\n",
      "transformer.h.8.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.8.post_attention_layernorm.bias torch.Size([1024])\n",
      "transformer.h.8.mlp.dense_h_to_4h.weight torch.Size([803, 1024])\n",
      "transformer.h.8.mlp.dense_h_to_4h.bias torch.Size([803])\n",
      "transformer.h.8.mlp.dense_4h_to_h.weight torch.Size([1024, 803])\n",
      "transformer.h.8.mlp.dense_4h_to_h.bias torch.Size([1024])\n",
      "transformer.h.9.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.9.input_layernorm.bias torch.Size([1024])\n",
      "transformer.h.9.self_attention.query_key_value.weight torch.Size([768, 1024])\n",
      "transformer.h.9.self_attention.query_key_value.bias torch.Size([768])\n",
      "transformer.h.9.self_attention.dense.weight torch.Size([1024, 256])\n",
      "transformer.h.9.self_attention.dense.bias torch.Size([1024])\n",
      "transformer.h.9.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.9.post_attention_layernorm.bias torch.Size([1024])\n",
      "transformer.h.9.mlp.dense_h_to_4h.weight torch.Size([809, 1024])\n",
      "transformer.h.9.mlp.dense_h_to_4h.bias torch.Size([809])\n",
      "transformer.h.9.mlp.dense_4h_to_h.weight torch.Size([1024, 809])\n",
      "transformer.h.9.mlp.dense_4h_to_h.bias torch.Size([1024])\n",
      "transformer.h.10.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.10.input_layernorm.bias torch.Size([1024])\n",
      "transformer.h.10.self_attention.query_key_value.weight torch.Size([768, 1024])\n",
      "transformer.h.10.self_attention.query_key_value.bias torch.Size([768])\n",
      "transformer.h.10.self_attention.dense.weight torch.Size([1024, 256])\n",
      "transformer.h.10.self_attention.dense.bias torch.Size([1024])\n",
      "transformer.h.10.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.10.post_attention_layernorm.bias torch.Size([1024])\n",
      "transformer.h.10.mlp.dense_h_to_4h.weight torch.Size([1536, 1024])\n",
      "transformer.h.10.mlp.dense_h_to_4h.bias torch.Size([1536])\n",
      "transformer.h.10.mlp.dense_4h_to_h.weight torch.Size([1024, 1536])\n",
      "transformer.h.10.mlp.dense_4h_to_h.bias torch.Size([1024])\n",
      "transformer.h.11.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.11.input_layernorm.bias torch.Size([1024])\n",
      "transformer.h.11.self_attention.query_key_value.weight torch.Size([768, 1024])\n",
      "transformer.h.11.self_attention.query_key_value.bias torch.Size([768])\n",
      "transformer.h.11.self_attention.dense.weight torch.Size([1024, 256])\n",
      "transformer.h.11.self_attention.dense.bias torch.Size([1024])\n",
      "transformer.h.11.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.11.post_attention_layernorm.bias torch.Size([1024])\n",
      "transformer.h.11.mlp.dense_h_to_4h.weight torch.Size([1527, 1024])\n",
      "transformer.h.11.mlp.dense_h_to_4h.bias torch.Size([1527])\n",
      "transformer.h.11.mlp.dense_4h_to_h.weight torch.Size([1024, 1527])\n",
      "transformer.h.11.mlp.dense_4h_to_h.bias torch.Size([1024])\n",
      "transformer.h.12.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.12.input_layernorm.bias torch.Size([1024])\n",
      "transformer.h.12.self_attention.query_key_value.weight torch.Size([768, 1024])\n",
      "transformer.h.12.self_attention.query_key_value.bias torch.Size([768])\n",
      "transformer.h.12.self_attention.dense.weight torch.Size([1024, 256])\n",
      "transformer.h.12.self_attention.dense.bias torch.Size([1024])\n",
      "transformer.h.12.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.12.post_attention_layernorm.bias torch.Size([1024])\n",
      "transformer.h.12.mlp.dense_h_to_4h.weight torch.Size([1817, 1024])\n",
      "transformer.h.12.mlp.dense_h_to_4h.bias torch.Size([1817])\n",
      "transformer.h.12.mlp.dense_4h_to_h.weight torch.Size([1024, 1817])\n",
      "transformer.h.12.mlp.dense_4h_to_h.bias torch.Size([1024])\n",
      "transformer.h.13.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.13.input_layernorm.bias torch.Size([1024])\n",
      "transformer.h.13.self_attention.query_key_value.weight torch.Size([768, 1024])\n",
      "transformer.h.13.self_attention.query_key_value.bias torch.Size([768])\n",
      "transformer.h.13.self_attention.dense.weight torch.Size([1024, 256])\n",
      "transformer.h.13.self_attention.dense.bias torch.Size([1024])\n",
      "transformer.h.13.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.13.post_attention_layernorm.bias torch.Size([1024])\n",
      "transformer.h.13.mlp.dense_h_to_4h.weight torch.Size([1766, 1024])\n",
      "transformer.h.13.mlp.dense_h_to_4h.bias torch.Size([1766])\n",
      "transformer.h.13.mlp.dense_4h_to_h.weight torch.Size([1024, 1766])\n",
      "transformer.h.13.mlp.dense_4h_to_h.bias torch.Size([1024])\n",
      "transformer.h.14.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.14.input_layernorm.bias torch.Size([1024])\n",
      "transformer.h.14.self_attention.query_key_value.weight torch.Size([960, 1024])\n",
      "transformer.h.14.self_attention.query_key_value.bias torch.Size([960])\n",
      "transformer.h.14.self_attention.dense.weight torch.Size([1024, 320])\n",
      "transformer.h.14.self_attention.dense.bias torch.Size([1024])\n",
      "transformer.h.14.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.14.post_attention_layernorm.bias torch.Size([1024])\n",
      "transformer.h.14.mlp.dense_h_to_4h.weight torch.Size([1833, 1024])\n",
      "transformer.h.14.mlp.dense_h_to_4h.bias torch.Size([1833])\n",
      "transformer.h.14.mlp.dense_4h_to_h.weight torch.Size([1024, 1833])\n",
      "transformer.h.14.mlp.dense_4h_to_h.bias torch.Size([1024])\n",
      "transformer.h.15.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.15.input_layernorm.bias torch.Size([1024])\n",
      "transformer.h.15.self_attention.query_key_value.weight torch.Size([768, 1024])\n",
      "transformer.h.15.self_attention.query_key_value.bias torch.Size([768])\n",
      "transformer.h.15.self_attention.dense.weight torch.Size([1024, 256])\n",
      "transformer.h.15.self_attention.dense.bias torch.Size([1024])\n",
      "transformer.h.15.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.15.post_attention_layernorm.bias torch.Size([1024])\n",
      "transformer.h.15.mlp.dense_h_to_4h.weight torch.Size([2120, 1024])\n",
      "transformer.h.15.mlp.dense_h_to_4h.bias torch.Size([2120])\n",
      "transformer.h.15.mlp.dense_4h_to_h.weight torch.Size([1024, 2120])\n",
      "transformer.h.15.mlp.dense_4h_to_h.bias torch.Size([1024])\n",
      "transformer.h.16.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.16.input_layernorm.bias torch.Size([1024])\n",
      "transformer.h.16.self_attention.query_key_value.weight torch.Size([768, 1024])\n",
      "transformer.h.16.self_attention.query_key_value.bias torch.Size([768])\n",
      "transformer.h.16.self_attention.dense.weight torch.Size([1024, 256])\n",
      "transformer.h.16.self_attention.dense.bias torch.Size([1024])\n",
      "transformer.h.16.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.16.post_attention_layernorm.bias torch.Size([1024])\n",
      "transformer.h.16.mlp.dense_h_to_4h.weight torch.Size([2577, 1024])\n",
      "transformer.h.16.mlp.dense_h_to_4h.bias torch.Size([2577])\n",
      "transformer.h.16.mlp.dense_4h_to_h.weight torch.Size([1024, 2577])\n",
      "transformer.h.16.mlp.dense_4h_to_h.bias torch.Size([1024])\n",
      "transformer.h.17.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.17.input_layernorm.bias torch.Size([1024])\n",
      "transformer.h.17.self_attention.query_key_value.weight torch.Size([768, 1024])\n",
      "transformer.h.17.self_attention.query_key_value.bias torch.Size([768])\n",
      "transformer.h.17.self_attention.dense.weight torch.Size([1024, 256])\n",
      "transformer.h.17.self_attention.dense.bias torch.Size([1024])\n",
      "transformer.h.17.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.17.post_attention_layernorm.bias torch.Size([1024])\n",
      "transformer.h.17.mlp.dense_h_to_4h.weight torch.Size([2910, 1024])\n",
      "transformer.h.17.mlp.dense_h_to_4h.bias torch.Size([2910])\n",
      "transformer.h.17.mlp.dense_4h_to_h.weight torch.Size([1024, 2910])\n",
      "transformer.h.17.mlp.dense_4h_to_h.bias torch.Size([1024])\n",
      "transformer.h.18.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.18.input_layernorm.bias torch.Size([1024])\n",
      "transformer.h.18.self_attention.query_key_value.weight torch.Size([960, 1024])\n",
      "transformer.h.18.self_attention.query_key_value.bias torch.Size([960])\n",
      "transformer.h.18.self_attention.dense.weight torch.Size([1024, 320])\n",
      "transformer.h.18.self_attention.dense.bias torch.Size([1024])\n",
      "transformer.h.18.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.18.post_attention_layernorm.bias torch.Size([1024])\n",
      "transformer.h.18.mlp.dense_h_to_4h.weight torch.Size([3265, 1024])\n",
      "transformer.h.18.mlp.dense_h_to_4h.bias torch.Size([3265])\n",
      "transformer.h.18.mlp.dense_4h_to_h.weight torch.Size([1024, 3265])\n",
      "transformer.h.18.mlp.dense_4h_to_h.bias torch.Size([1024])\n",
      "transformer.h.19.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.19.input_layernorm.bias torch.Size([1024])\n",
      "transformer.h.19.self_attention.query_key_value.weight torch.Size([768, 1024])\n",
      "transformer.h.19.self_attention.query_key_value.bias torch.Size([768])\n",
      "transformer.h.19.self_attention.dense.weight torch.Size([1024, 256])\n",
      "transformer.h.19.self_attention.dense.bias torch.Size([1024])\n",
      "transformer.h.19.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.19.post_attention_layernorm.bias torch.Size([1024])\n",
      "transformer.h.19.mlp.dense_h_to_4h.weight torch.Size([3510, 1024])\n",
      "transformer.h.19.mlp.dense_h_to_4h.bias torch.Size([3510])\n",
      "transformer.h.19.mlp.dense_4h_to_h.weight torch.Size([1024, 3510])\n",
      "transformer.h.19.mlp.dense_4h_to_h.bias torch.Size([1024])\n",
      "transformer.h.20.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.20.input_layernorm.bias torch.Size([1024])\n",
      "transformer.h.20.self_attention.query_key_value.weight torch.Size([2112, 1024])\n",
      "transformer.h.20.self_attention.query_key_value.bias torch.Size([2112])\n",
      "transformer.h.20.self_attention.dense.weight torch.Size([1024, 704])\n",
      "transformer.h.20.self_attention.dense.bias torch.Size([1024])\n",
      "transformer.h.20.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.20.post_attention_layernorm.bias torch.Size([1024])\n",
      "transformer.h.20.mlp.dense_h_to_4h.weight torch.Size([3299, 1024])\n",
      "transformer.h.20.mlp.dense_h_to_4h.bias torch.Size([3299])\n",
      "transformer.h.20.mlp.dense_4h_to_h.weight torch.Size([1024, 3299])\n",
      "transformer.h.20.mlp.dense_4h_to_h.bias torch.Size([1024])\n",
      "transformer.h.21.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.21.input_layernorm.bias torch.Size([1024])\n",
      "transformer.h.21.self_attention.query_key_value.weight torch.Size([3072, 1024])\n",
      "transformer.h.21.self_attention.query_key_value.bias torch.Size([3072])\n",
      "transformer.h.21.self_attention.dense.weight torch.Size([1024, 1024])\n",
      "transformer.h.21.self_attention.dense.bias torch.Size([1024])\n",
      "transformer.h.21.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.21.post_attention_layernorm.bias torch.Size([1024])\n",
      "transformer.h.21.mlp.dense_h_to_4h.weight torch.Size([3642, 1024])\n",
      "transformer.h.21.mlp.dense_h_to_4h.bias torch.Size([3642])\n",
      "transformer.h.21.mlp.dense_4h_to_h.weight torch.Size([1024, 3642])\n",
      "transformer.h.21.mlp.dense_4h_to_h.bias torch.Size([1024])\n",
      "transformer.h.22.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.22.input_layernorm.bias torch.Size([1024])\n",
      "transformer.h.22.self_attention.query_key_value.weight torch.Size([3072, 1024])\n",
      "transformer.h.22.self_attention.query_key_value.bias torch.Size([3072])\n",
      "transformer.h.22.self_attention.dense.weight torch.Size([1024, 1024])\n",
      "transformer.h.22.self_attention.dense.bias torch.Size([1024])\n",
      "transformer.h.22.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.22.post_attention_layernorm.bias torch.Size([1024])\n",
      "transformer.h.22.mlp.dense_h_to_4h.weight torch.Size([2615, 1024])\n",
      "transformer.h.22.mlp.dense_h_to_4h.bias torch.Size([2615])\n",
      "transformer.h.22.mlp.dense_4h_to_h.weight torch.Size([1024, 2615])\n",
      "transformer.h.22.mlp.dense_4h_to_h.bias torch.Size([1024])\n",
      "transformer.h.23.input_layernorm.weight torch.Size([1024])\n",
      "transformer.h.23.input_layernorm.bias torch.Size([1024])\n",
      "transformer.h.23.self_attention.query_key_value.weight torch.Size([768, 1024])\n",
      "transformer.h.23.self_attention.query_key_value.bias torch.Size([768])\n",
      "transformer.h.23.self_attention.dense.weight torch.Size([1024, 256])\n",
      "transformer.h.23.self_attention.dense.bias torch.Size([1024])\n",
      "transformer.h.23.post_attention_layernorm.weight torch.Size([1024])\n",
      "transformer.h.23.post_attention_layernorm.bias torch.Size([1024])\n",
      "transformer.h.23.mlp.dense_h_to_4h.weight torch.Size([781, 1024])\n",
      "transformer.h.23.mlp.dense_h_to_4h.bias torch.Size([781])\n",
      "transformer.h.23.mlp.dense_4h_to_h.weight torch.Size([1024, 781])\n",
      "transformer.h.23.mlp.dense_4h_to_h.bias torch.Size([1024])\n",
      "transformer.ln_f.weight torch.Size([1024])\n",
      "transformer.ln_f.bias torch.Size([1024])\n",
      "lm_head.weight torch.Size([250880, 1024])\n"
     ]
    }
   ],
   "source": [
    "num_params_now = 0\n",
    "all_num_heads = []\n",
    "for param_name in pruned_model_params.keys():\n",
    "    shape = pruned_model_params[param_name].shape\n",
    "    state_dict_shapes[param_name] = shape\n",
    "    print(param_name, shape)\n",
    "    \n",
    "    if \"self_attention.query_key_value.bias\" in param_name:\n",
    "        num_heads = shape[0] // 3 // head_dim\n",
    "        all_num_heads.append(num_heads)\n",
    "    \n",
    "    if param_name == \"lm_head.weight\":\n",
    "        continue\n",
    "    elif len(shape) == 1:\n",
    "        num_params_now += shape[0]\n",
    "    else:\n",
    "        num_params_now += (shape[0] * shape[1])\n",
    "    \n",
    "pkl.dump((state_dict_shapes, all_num_heads), open(\"pruned_30percent_560m_bloom_state_dict_shapes.pkl\", \"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "05cdae0e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6909103884756999"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_params_now / 559214592"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "fdbaf51b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 4, 4, 4, 5, 4, 11, 16, 16, 4]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_num_heads"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "28af6ba2-5c54-4693-a1ba-56cc2d8da64b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5639706566220238"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "node_num / (len(all_nodes) + (24 * 1024 * 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "81c1d532-4970-4891-88a0-b1110c867364",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "97021"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "node_num"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "beff6b26-2dd6-4776-8cd7-d7551442b756",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cc17e958",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4b6f5541",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ade382bd",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tap",
   "language": "python",
   "name": "tap"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}