test_blender_bot-Copy1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c48024fa-d8ae-42de-9caa-ec99e01d81d0",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/homebrew/lib/python3.10/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from transformers import AutoTokenizer, BlenderbotForConditionalGeneration\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "056c3065-6475-4d81-8945-c39fc787f5c0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Human: I'm looking for someone to talk to. Life can be so lonely sometimes and it helps to have someone to vent to.\n"
     ]
    }
   ],
   "source": [
    "mname = \"facebook/blenderbot-400M-distill\"\n",
    "weights_path = \"pruned_400m_blender_bot2.pt\"\n",
    "\n",
    "model = BlenderbotForConditionalGeneration.from_pretrained(mname)\n",
    "tokenizer = AutoTokenizer.from_pretrained(mname)\n",
    "UTTERANCE = \"I'm looking for someone to talk to. Life can be so lonely sometimes and it helps to have someone to vent to.\"\n",
    "print(\"Human:\", UTTERANCE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "90e8d54b-688a-4d75-a243-51774b848fa0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bot:  I'm sorry to hear that. I'm here for you if you need someone to chat with.\n"
     ]
    }
   ],
   "source": [
    "inputs = tokenizer([UTTERANCE], return_tensors=\"pt\")\n",
    "reply_ids = model.generate(\n",
    "    input_ids=inputs[\"input_ids\"], \n",
    "    max_new_tokens=100, \n",
    "    do_sample=False, \n",
    "    top_k=50, \n",
    "    top_p=0.95,\n",
    ")\n",
    "print(\"Bot:\", tokenizer.batch_decode(reply_ids, skip_special_tokens=True)[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "821274fb-57d6-4ceb-9d66-d90ada1e77c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers.models.blenderbot.configuration_blenderbot import BlenderbotConfig\n",
    "from pruning.pruned_blender_bot import PrunedBlenderbotForConditionalGeneration\n",
    "\n",
    "weights_path = \"pruned_400m_blender_bot2.pt\"\n",
    "state_dict_shapes_path = \"pruned_400m_blender_bot2_state_dict_shapes.pkl\"\n",
    "\n",
    "blenderbot_config = BlenderbotConfig(\n",
    "    vocab_size=8008,\n",
    "    max_position_embeddings=128,\n",
    "    encoder_layers=2,\n",
    "    encoder_ffn_dim=5120,\n",
    "    encoder_attention_heads=32,\n",
    "    decoder_layers=12,\n",
    "    decoder_ffn_dim=5120,\n",
    "    decoder_attention_heads=32,\n",
    "    encoder_layerdrop=0.0,\n",
    "    decoder_layerdrop=0.0,\n",
    "    layernorm_variant=\"prelayernorm\",\n",
    "    use_cache=True,\n",
    "    is_encoder_decoder=True,\n",
    "    activation_function=\"gelu\",\n",
    "    d_model=1280,\n",
    "    dropout=0.1,\n",
    "    attention_dropout=0.0,\n",
    "    activation_dropout=0.0,\n",
    "    init_std=0.02,\n",
    "    decoder_start_token_id=1,\n",
    "    scale_embedding=False,\n",
    "    pad_token_id=0,\n",
    "    bos_token_id=1,\n",
    "    eos_token_id=2,\n",
    "    encoder_no_repeat_ngram_size=3,\n",
    "    forced_eos_token_id=2,\n",
    "    num_beams=10,\n",
    "    length_penalty=65,\n",
    "    min_length=20,\n",
    "    static_position_embeddings=False,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b0f7ae8a-5641-4765-9f99-e059ce14458d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#pruned_model = PrunedBlenderbotForConditionalGeneration(blenderbot_config, state_dict_shapes_path)\n",
    "pruned_model = BlenderbotForConditionalGeneration(blenderbot_config)\n",
    "pruned_model.load_state_dict(torch.load(weights_path))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f90642a3-467a-4c07-83d5-ed6d545bb501",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BlenderbotConfig {\n",
       "  \"_name_or_path\": \"facebook/blenderbot-400M-distill\",\n",
       "  \"activation_dropout\": 0.0,\n",
       "  \"activation_function\": \"gelu\",\n",
       "  \"add_bias_logits\": false,\n",
       "  \"add_final_layer_norm\": true,\n",
       "  \"architectures\": [\n",
       "    \"BlenderbotForConditionalGeneration\"\n",
       "  ],\n",
       "  \"attention_dropout\": 0.0,\n",
       "  \"bos_token_id\": 1,\n",
       "  \"classif_dropout\": 0.0,\n",
       "  \"classifier_dropout\": 0.0,\n",
       "  \"d_model\": 1280,\n",
       "  \"decoder_attention_heads\": 32,\n",
       "  \"decoder_ffn_dim\": 5120,\n",
       "  \"decoder_layerdrop\": 0.0,\n",
       "  \"decoder_layers\": 12,\n",
       "  \"decoder_start_token_id\": 1,\n",
       "  \"do_blenderbot_90_layernorm\": true,\n",
       "  \"dropout\": 0.1,\n",
       "  \"encoder_attention_heads\": 32,\n",
       "  \"encoder_ffn_dim\": 5120,\n",
       "  \"encoder_layerdrop\": 0.0,\n",
       "  \"encoder_layers\": 2,\n",
       "  \"encoder_no_repeat_ngram_size\": 3,\n",
       "  \"eos_token_id\": 2,\n",
       "  \"extra_layer_norm\": false,\n",
       "  \"extra_pos_embeddings\": 0,\n",
       "  \"force_bos_token_to_be_generated\": false,\n",
       "  \"forced_eos_token_id\": 2,\n",
       "  \"gradient_checkpointing\": false,\n",
       "  \"id2label\": {\n",
       "    \"0\": \"LABEL_0\",\n",
       "    \"1\": \"LABEL_1\",\n",
       "    \"2\": \"LABEL_2\"\n",
       "  },\n",
       "  \"init_std\": 0.02,\n",
       "  \"is_encoder_decoder\": true,\n",
       "  \"label2id\": {\n",
       "    \"LABEL_0\": 0,\n",
       "    \"LABEL_1\": 1,\n",
       "    \"LABEL_2\": 2\n",
       "  },\n",
       "  \"layernorm_variant\": \"prelayernorm\",\n",
       "  \"length_penalty\": 0.65,\n",
       "  \"max_length\": 60,\n",
       "  \"max_position_embeddings\": 128,\n",
       "  \"min_length\": 20,\n",
       "  \"model_type\": \"blenderbot\",\n",
       "  \"no_repeat_ngram_size\": 3,\n",
       "  \"normalize_before\": true,\n",
       "  \"normalize_embedding\": false,\n",
       "  \"num_beams\": 10,\n",
       "  \"num_hidden_layers\": 2,\n",
       "  \"pad_token_id\": 0,\n",
       "  \"scale_embedding\": true,\n",
       "  \"static_position_embeddings\": false,\n",
       "  \"transformers_version\": \"4.26.0\",\n",
       "  \"unk_token_id\": 3,\n",
       "  \"use_cache\": true,\n",
       "  \"vocab_size\": 8008\n",
       "}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "fa30600d-5cb7-4558-b1cc-9ed3538cec9e",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_config = dir(model.config)\n",
    "pruned_model_config = set(dir(pruned_model.config))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "8ed5b4cf-902c-414e-82db-30bcb7b7cd7c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "add_bias_logits\n",
      "add_final_layer_norm\n",
      "classif_dropout\n",
      "classifier_dropout\n",
      "do_blenderbot_90_layernorm\n",
      "extra_layer_norm\n",
      "extra_pos_embeddings\n",
      "force_bos_token_to_be_generated\n",
      "gradient_checkpointing\n",
      "normalize_before\n",
      "normalize_embedding\n",
      "unk_token_id\n"
     ]
    }
   ],
   "source": [
    "for key in model_config:\n",
    "    if key not in pruned_model_config:\n",
    "        print(key)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d377a81-4103-464c-a481-037e04908084",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(model.config)\n",
    "print(pruned_model.config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "89bb7407-84b2-45be-a184-f13e56954500",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bot: . to. to to to to to to to a to to to to to to to to to to to to to to to to to to to to to to and to to to to to to to and to and and to to to to and to to to to to to to to to to to to to to to to to to to to to to to. of to to to to to to in to to to to. to to. and to to to to to to to\n"
     ]
    }
   ],
   "source": [
    "reply_ids = pruned_model.generate(\n",
    "    input_ids=inputs[\"input_ids\"], \n",
    "    max_new_tokens=100, \n",
    "    do_sample=False, \n",
    "    top_k=50, \n",
    "    top_p=0.95,\n",
    ")\n",
    "print(\"Bot:\", tokenizer.batch_decode(reply_ids, skip_special_tokens=True)[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "9e2cc214-d923-4d54-9517-5133d1f700c4",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OrderedDict([('final_logits_bias', tensor([[0., 0., 0.,  ..., 0., 0., 0.]])), ('model.shared.weight', tensor([[ 0.0117, -0.0016, -0.0037,  ...,  0.0047, -0.0064, -0.0061],\n",
      "        [-0.0008, -0.0297,  0.0005,  ..., -0.0311,  0.0016, -0.0305],\n",
      "        [-0.0167,  0.0146, -0.0097,  ..., -0.0175,  0.0177, -0.0497],\n",
      "        ...,\n",
      "        [ 0.0285, -0.0268,  0.0322,  ...,  0.0152,  0.0374, -0.0690],\n",
      "        [-0.0064, -0.0241,  0.0508,  ..., -0.0005,  0.0751, -0.0540],\n",
      "        [ 0.0312,  0.0444,  0.0149,  ...,  0.0448,  0.0242, -0.0059]])), ('model.encoder.embed_tokens.weight', tensor([[ 0.0117, -0.0016, -0.0037,  ...,  0.0047, -0.0064, -0.0061],\n",
      "        [-0.0008, -0.0297,  0.0005,  ..., -0.0311,  0.0016, -0.0305],\n",
      "        [-0.0167,  0.0146, -0.0097,  ..., -0.0175,  0.0177, -0.0497],\n",
      "        ...,\n",
      "        [ 0.0285, -0.0268,  0.0322,  ...,  0.0152,  0.0374, -0.0690],\n",
      "        [-0.0064, -0.0241,  0.0508,  ..., -0.0005,  0.0751, -0.0540],\n",
      "        [ 0.0312,  0.0444,  0.0149,  ...,  0.0448,  0.0242, -0.0059]])), ('model.encoder.embed_positions.weight', tensor([[ 0.0000e+00,  1.0000e+00,  0.0000e+00,  ...,  1.0000e+00,\n",
      "          0.0000e+00,  1.0000e+00],\n",
      "        [ 8.4131e-01,  5.4053e-01,  8.3350e-01,  ...,  1.0000e+00,\n",
      "          1.0145e-04,  1.0000e+00],\n",
      "        [ 9.0918e-01, -4.1626e-01,  9.2090e-01,  ...,  1.0000e+00,\n",
      "          2.0289e-04,  1.0000e+00],\n",
      "        ...,\n",
      "        [-6.1621e-01,  7.8760e-01, -6.3818e-01,  ...,  1.0000e+00,\n",
      "          1.2680e-02,  1.0000e+00],\n",
      "        [ 3.3008e-01,  9.4385e-01, -9.9414e-01,  ...,  1.0000e+00,\n",
      "          1.2779e-02,  1.0000e+00],\n",
      "        [ 9.7266e-01,  2.3230e-01, -4.6021e-01,  ...,  1.0000e+00,\n",
      "          1.2886e-02,  1.0000e+00]])), ('model.encoder.layers.0.self_attn.k_proj.weight', tensor([[-1.4252e-02,  1.3901e-02,  2.2640e-03,  ...,  2.2873e-02,\n",
      "          2.7176e-02, -2.8702e-02],\n",
      "        [ 6.3477e-02,  1.4305e-02,  4.8035e-02,  ...,  1.9928e-02,\n",
      "          3.0270e-03,  1.2924e-02],\n",
      "        [ 5.3009e-02, -1.4259e-02,  5.0446e-02,  ...,  1.2993e-02,\n",
      "          3.5210e-03, -2.2842e-02],\n",
      "        ...,\n",
      "        [-6.1310e-02, -5.9128e-03, -4.0649e-02,  ...,  3.9978e-02,\n",
      "          2.9587e-02, -2.2278e-02],\n",
      "        [-1.4694e-02,  9.0420e-05, -6.1188e-02,  ...,  6.0997e-03,\n",
      "         -1.2016e-02,  1.9638e-02],\n",
      "        [ 2.5070e-02, -6.0974e-02, -9.3994e-02,  ..., -3.4790e-02,\n",
      "         -5.8823e-03, -1.1292e-02]])), ('model.encoder.layers.0.self_attn.k_proj.bias', tensor([ 0.6875,  0.5327,  0.0160,  ...,  0.0029, -0.2625,  0.4897])), ('model.encoder.layers.0.self_attn.v_proj.weight', tensor([[ 0.0434,  0.0158, -0.0113,  ...,  0.1140,  0.0315, -0.1212],\n",
      "        [ 0.0598,  0.0317, -0.0165,  ...,  0.0549, -0.0067, -0.1024],\n",
      "        [ 0.0670, -0.0820,  0.0233,  ...,  0.0870,  0.0266,  0.0873],\n",
      "        ...,\n",
      "        [ 0.0596,  0.0439,  0.0205,  ..., -0.0423,  0.0158, -0.0385],\n",
      "        [ 0.1506,  0.0390,  0.1052,  ..., -0.0245, -0.0195, -0.0009],\n",
      "        [-0.0542,  0.0770, -0.0568,  ..., -0.0226,  0.0010, -0.0314]])), ('model.encoder.layers.0.self_attn.v_proj.bias', tensor([-0.1428, -0.0440,  0.0391,  ...,  0.0039, -0.1434, -0.0905])), ('model.encoder.layers.0.self_attn.q_proj.weight', tensor([[-0.0022, -0.0158, -0.0185,  ...,  0.0215, -0.0173, -0.0095],\n",
      "        [-0.0479, -0.0364, -0.0199,  ...,  0.0351,  0.0086, -0.0002],\n",
      "        [-0.0479,  0.0207,  0.0074,  ..., -0.0006, -0.0075, -0.0113],\n",
      "        ...,\n",
      "        [-0.0456, -0.0097,  0.0035,  ...,  0.0357,  0.0173, -0.0274],\n",
      "        [ 0.0274, -0.0876,  0.0401,  ...,  0.0159, -0.0059,  0.0035],\n",
      "        [ 0.1534, -0.0584,  0.1360,  ..., -0.0146, -0.0003, -0.0249]])), ('model.encoder.layers.0.self_attn.q_proj.bias', tensor([-0.1016, -0.0593, -0.0230,  ...,  0.0084, -0.0441,  0.1791])), ('model.encoder.layers.0.self_attn.out_proj.weight', tensor([[-0.0004,  0.0218, -0.0133,  ..., -0.0267,  0.0245,  0.0451],\n",
      "        [ 0.0246, -0.0578,  0.0115,  ...,  0.0188, -0.0199,  0.0209],\n",
      "        [ 0.0494,  0.0443, -0.0507,  ..., -0.0411,  0.0102,  0.0223],\n",
      "        ...,\n",
      "        [ 0.0253,  0.0558,  0.0114,  ...,  0.0718,  0.0811,  0.0653],\n",
      "        [ 0.0483, -0.0825, -0.0918,  ..., -0.0638,  0.0570, -0.0209],\n",
      "        [-0.0597,  0.1008,  0.0086,  ...,  0.0317,  0.0059,  0.0465]])), ('model.encoder.layers.0.self_attn.out_proj.bias', tensor([ 0.0060, -0.0479,  0.0698,  ...,  0.1161, -0.0925,  0.1481])), ('model.encoder.layers.0.self_attn_layer_norm.weight', tensor([0.7305, 0.7261, 0.8901,  ..., 0.6157, 0.7666, 0.8472])), ('model.encoder.layers.0.self_attn_layer_norm.bias', tensor([ 0.0939,  0.0862,  0.0952,  ...,  0.0544, -0.1621,  0.1478])), ('model.encoder.layers.0.fc1.weight', tensor([[-0.1110, -0.1617,  0.0451,  ...,  0.0222,  0.0153, -0.0504],\n",
      "        [-0.0671, -0.1305, -0.1423,  ..., -0.0055,  0.1201, -0.0287],\n",
      "        [ 0.0275,  0.0009, -0.1223,  ..., -0.0525,  0.0542, -0.0635],\n",
      "        ...,\n",
      "        [ 0.0380,  0.0110, -0.0417,  ...,  0.0303,  0.1245,  0.0651],\n",
      "        [-0.0525, -0.0485, -0.0963,  ...,  0.0399,  0.0131, -0.0213],\n",
      "        [-0.2524, -0.0080, -0.1517,  ...,  0.0596,  0.2196, -0.0667]])), ('model.encoder.layers.0.fc1.bias', tensor([-0.0579, -0.0899, -0.1043,  ..., -0.0679,  0.0443, -0.0933])), ('model.encoder.layers.0.fc2.weight', tensor([[ 0.0972, -0.0290, -0.0766,  ..., -0.0288,  0.0296,  0.0721],\n",
      "        [-0.1396,  0.0359, -0.0408,  ...,  0.0328,  0.0749, -0.0170],\n",
      "        [ 0.0908,  0.0154,  0.0894,  ...,  0.0150,  0.0334, -0.0603],\n",
      "        ...,\n",
      "        [-0.0969, -0.0117,  0.1708,  ..., -0.0453, -0.0170, -0.0870],\n",
      "        [-0.0620, -0.0434, -0.1857,  ..., -0.0644, -0.0546, -0.0995],\n",
      "        [-0.0631,  0.0715,  0.0532,  ..., -0.0329,  0.0636,  0.0293]])), ('model.encoder.layers.0.fc2.bias', tensor([-0.0303,  0.0335, -0.0441,  ...,  0.0266,  0.0568,  0.1160])), ('model.encoder.layers.0.final_layer_norm.weight', tensor([0.9619, 0.9580, 0.9976,  ..., 0.9785, 0.9736, 0.9971])), ('model.encoder.layers.0.final_layer_norm.bias', tensor([ 0.0829,  0.0543,  0.0828,  ...,  0.1273, -0.0426,  0.2529])), ('model.encoder.layers.1.self_attn.k_proj.weight', tensor([[ 0.0510, -0.0385, -0.0290,  ...,  0.0351,  0.1009, -0.0405],\n",
      "        [-0.0289,  0.1055,  0.0956,  ...,  0.0309,  0.0180,  0.0010],\n",
      "        [ 0.1075, -0.0335, -0.0012,  ...,  0.0270,  0.0039,  0.0160],\n",
      "        ...,\n",
      "        [ 0.0435, -0.0140,  0.0804,  ..., -0.0136, -0.0834,  0.0152],\n",
      "        [ 0.0312,  0.0646,  0.0122,  ..., -0.0232, -0.0050,  0.0573],\n",
      "        [ 0.0238, -0.0445, -0.0475,  ...,  0.0078, -0.0294,  0.0680]])), ('model.encoder.layers.1.self_attn.k_proj.bias', tensor([ 0.2734, -0.0634,  0.0418,  ..., -0.1788, -0.1479, -0.5220])), ('model.encoder.layers.1.self_attn.v_proj.weight', tensor([[ 0.1520,  0.0017, -0.0407,  ...,  0.1087, -0.1799,  0.0628],\n",
      "        [-0.0219, -0.1039, -0.0315,  ..., -0.1207,  0.0256, -0.1544],\n",
      "        [ 0.0641, -0.0442,  0.0260,  ...,  0.0110,  0.0592, -0.0687],\n",
      "        ...,\n",
      "        [-0.0253, -0.0416, -0.0369,  ...,  0.0579,  0.0779,  0.1241],\n",
      "        [ 0.1823,  0.0208, -0.0528,  ..., -0.1475,  0.1042, -0.1014],\n",
      "        [ 0.0723,  0.1906,  0.0851,  ..., -0.1250,  0.0954, -0.1200]])), ('model.encoder.layers.1.self_attn.v_proj.bias', tensor([-0.0304, -0.0051, -0.0031,  ...,  0.0520,  0.0323, -0.0044])), ('model.encoder.layers.1.self_attn.q_proj.weight', tensor([[ 0.0089, -0.0491, -0.0609,  ...,  0.0582, -0.0094,  0.0388],\n",
      "        [-0.1129, -0.1174,  0.0185,  ...,  0.0902, -0.0132,  0.0309],\n",
      "        [-0.0132,  0.0087,  0.0388,  ..., -0.0137, -0.0078, -0.0712],\n",
      "        ...,\n",
      "        [ 0.0155,  0.0962,  0.0027,  ...,  0.0036,  0.0292,  0.0549],\n",
      "        [ 0.0468,  0.0305,  0.0651,  ..., -0.0940,  0.0602, -0.0098],\n",
      "        [ 0.0105, -0.1140, -0.0341,  ...,  0.0235,  0.1104,  0.0443]])), ('model.encoder.layers.1.self_attn.q_proj.bias', tensor([-0.0504, -0.0009,  0.0266,  ..., -0.0433,  0.1132,  0.0113])), ('model.encoder.layers.1.self_attn.out_proj.weight', tensor([[-8.8989e-02,  6.3904e-02, -5.9601e-02,  ..., -2.5925e-02,\n",
      "          7.9250e-04,  1.7881e-05],\n",
      "        [-1.5552e-01,  3.7354e-02, -1.0094e-02,  ..., -6.3660e-02,\n",
      "          3.8177e-02,  3.3722e-02],\n",
      "        [ 5.2185e-02,  5.8044e-02,  2.7405e-02,  ..., -9.3079e-02,\n",
      "          8.6182e-02,  6.4514e-02],\n",
      "        ...,\n",
      "        [ 4.1473e-02, -8.2947e-02, -5.4382e-02,  ...,  5.1270e-02,\n",
      "         -6.1615e-02,  3.3436e-03],\n",
      "        [ 2.8931e-02,  5.6366e-02, -4.5166e-02,  ..., -1.7548e-02,\n",
      "          6.7444e-02,  8.8867e-02],\n",
      "        [ 2.6520e-02,  5.3864e-02,  3.3630e-02,  ..., -5.7556e-02,\n",
      "         -2.6642e-02,  4.2450e-02]])), ('model.encoder.layers.1.self_attn.out_proj.bias', tensor([-0.0119, -0.0701,  0.0304,  ...,  0.0527, -0.0335, -0.0072])), ('model.encoder.layers.1.self_attn_layer_norm.weight', tensor([0.9297, 0.9209, 1.0000,  ..., 0.9878, 1.0000, 0.9980])), ('model.encoder.layers.1.self_attn_layer_norm.bias', tensor([ 0.0181,  0.0452,  0.0309,  ..., -0.0314, -0.0093,  0.0179])), ('model.encoder.layers.1.fc1.weight', tensor([[ 0.1049, -0.0697, -0.1120,  ..., -0.0665,  0.0113,  0.0679],\n",
      "        [-0.0021, -0.0074,  0.0032,  ..., -0.0317,  0.1844, -0.0156],\n",
      "        [-0.0352,  0.0102,  0.0361,  ..., -0.1263,  0.0362,  0.0986],\n",
      "        ...,\n",
      "        [-0.0151,  0.0258,  0.0235,  ...,  0.0856,  0.0417,  0.0292],\n",
      "        [ 0.0291, -0.0777,  0.0193,  ...,  0.0399,  0.0597,  0.0281],\n",
      "        [-0.0184,  0.0049,  0.0012,  ...,  0.0844,  0.0825, -0.0399]])), ('model.encoder.layers.1.fc1.bias', tensor([-0.1664,  0.0275,  0.0437,  ...,  0.0336, -0.0846,  0.0454])), ('model.encoder.layers.1.fc2.weight', tensor([[ 0.0436,  0.0857,  0.0392,  ..., -0.0236, -0.0107, -0.0218],\n",
      "        [-0.1877,  0.0112,  0.0920,  ..., -0.0257, -0.1133,  0.0235],\n",
      "        [-0.0438,  0.0085,  0.1069,  ..., -0.0365,  0.0093,  0.0210],\n",
      "        ...,\n",
      "        [-0.1073,  0.0314,  0.0787,  ..., -0.1395, -0.2090, -0.0189],\n",
      "        [ 0.0309,  0.0684,  0.0050,  ..., -0.0121, -0.0648, -0.0209],\n",
      "        [ 0.1009, -0.0157, -0.0353,  ..., -0.0151,  0.1054,  0.0393]])), ('model.encoder.layers.1.fc2.bias', tensor([-0.0366,  0.0397, -0.0044,  ...,  0.0154, -0.0348,  0.0625])), ('model.encoder.layers.1.final_layer_norm.weight', tensor([0.9731, 0.9966, 0.9897,  ..., 0.9785, 0.9966, 0.9990])), ('model.encoder.layers.1.final_layer_norm.bias', tensor([-0.0354,  0.0622,  0.0493,  ...,  0.0172, -0.0144, -0.0407])), ('model.encoder.layer_norm.weight', tensor([0.8296, 0.7988, 0.9453,  ..., 0.9028, 0.9121, 0.9780])), ('model.encoder.layer_norm.bias', tensor([ 1.2789e-03,  9.4223e-04, -3.0398e-05,  ..., -3.3588e-03,\n",
      "         1.3306e-02,  2.9053e-02])), ('model.decoder.embed_tokens.weight', tensor([[ 0.0117, -0.0016, -0.0037,  ...,  0.0047, -0.0064, -0.0061],\n",
      "        [-0.0008, -0.0297,  0.0005,  ..., -0.0311,  0.0016, -0.0305],\n",
      "        [-0.0167,  0.0146, -0.0097,  ..., -0.0175,  0.0177, -0.0497],\n",
      "        ...,\n",
      "        [ 0.0285, -0.0268,  0.0322,  ...,  0.0152,  0.0374, -0.0690],\n",
      "        [-0.0064, -0.0241,  0.0508,  ..., -0.0005,  0.0751, -0.0540],\n",
      "        [ 0.0312,  0.0444,  0.0149,  ...,  0.0448,  0.0242, -0.0059]])), ('model.decoder.embed_positions.weight', tensor([[ 0.0000e+00,  1.0000e+00,  0.0000e+00,  ...,  1.0000e+00,\n",
      "          0.0000e+00,  1.0000e+00],\n",
      "        [ 8.4131e-01,  5.4053e-01,  8.3350e-01,  ...,  1.0000e+00,\n",
      "          1.0145e-04,  1.0000e+00],\n",
      "        [ 9.0918e-01, -4.1626e-01,  9.2090e-01,  ...,  1.0000e+00,\n",
      "          2.0289e-04,  1.0000e+00],\n",
      "        ...,\n",
      "        [-6.1621e-01,  7.8760e-01, -6.3818e-01,  ...,  1.0000e+00,\n",
      "          1.2680e-02,  1.0000e+00],\n",
      "        [ 3.3008e-01,  9.4385e-01, -9.9414e-01,  ...,  1.0000e+00,\n",
      "          1.2779e-02,  1.0000e+00],\n",
      "        [ 9.7266e-01,  2.3230e-01, -4.6021e-01,  ...,  1.0000e+00,\n",
      "          1.2886e-02,  1.0000e+00]])), ('model.decoder.layers.0.self_attn.k_proj.weight', tensor([[ 0.0484,  0.0712, -0.0847,  ...,  0.0782,  0.0122, -0.0386],\n",
      "        [-0.0462,  0.0854, -0.0665,  ..., -0.0100, -0.0510,  0.0535],\n",
      "        [-0.1028, -0.0244,  0.0502,  ..., -0.0248,  0.0080,  0.0351],\n",
      "        ...,\n",
      "        [ 0.0238, -0.0105, -0.0453,  ..., -0.0129, -0.0126, -0.0457],\n",
      "        [-0.0717,  0.0232,  0.1180,  ..., -0.0329,  0.0762, -0.1298],\n",
      "        [-0.0285,  0.0214, -0.0948,  ...,  0.0366, -0.0384,  0.0197]])), ('model.decoder.layers.0.self_attn.k_proj.bias', tensor([-0.1733,  0.4946,  0.5347,  ..., -1.0215,  0.5122, -0.8730])), ('model.decoder.layers.0.self_attn.v_proj.weight', tensor([[ 0.0678, -0.0625, -0.0201,  ...,  0.0405, -0.0453, -0.0753],\n",
      "        [-0.0395, -0.0141, -0.0752,  ...,  0.0052, -0.0428,  0.0868],\n",
      "        [-0.0247, -0.0069,  0.1147,  ...,  0.0002, -0.0570,  0.0052],\n",
      "        ...,\n",
      "        [ 0.0947,  0.0436, -0.0009,  ..., -0.0132,  0.0635, -0.0112],\n",
      "        [-0.0420, -0.0050, -0.0715,  ...,  0.0585,  0.0701,  0.0974],\n",
      "        [-0.0188, -0.0697, -0.0780,  ...,  0.0127,  0.0795,  0.0968]])), ('model.decoder.layers.0.self_attn.v_proj.bias', tensor([-0.0613, -0.0275, -0.0696,  ...,  0.0644, -0.0762,  0.0574])), ('model.decoder.layers.0.self_attn.q_proj.weight', tensor([[-0.0178,  0.0311, -0.0081,  ..., -0.0034,  0.0455, -0.0608],\n",
      "        [ 0.0360, -0.0297, -0.0272,  ...,  0.0354,  0.0499, -0.0556],\n",
      "        [-0.0144, -0.0150, -0.0165,  ...,  0.0448,  0.0945, -0.0554],\n",
      "        ...,\n",
      "        [-0.0076,  0.0231,  0.0348,  ...,  0.0626,  0.0105, -0.0116],\n",
      "        [ 0.0229,  0.0051, -0.0056,  ..., -0.0507,  0.0511, -0.0023],\n",
      "        [ 0.0123,  0.0885,  0.0749,  ...,  0.0462,  0.1035, -0.0107]])), ('model.decoder.layers.0.self_attn.q_proj.bias', tensor([ 0.1990, -0.1849,  0.0450,  ...,  0.1223, -0.1154,  0.2659])), ('model.decoder.layers.0.self_attn.out_proj.weight', tensor([[ 0.0292, -0.1025, -0.0174,  ...,  0.0438,  0.0130, -0.0049],\n",
      "        [-0.0066,  0.1279,  0.0651,  ...,  0.0295,  0.0471,  0.0383],\n",
      "        [ 0.0443,  0.0170,  0.0719,  ...,  0.0309,  0.0804,  0.0202],\n",
      "        ...,\n",
      "        [-0.1484,  0.0398, -0.0307,  ...,  0.0294,  0.0130,  0.0201],\n",
      "        [ 0.0190,  0.0120, -0.0205,  ..., -0.1299, -0.0183, -0.1092],\n",
      "        [ 0.1045,  0.0531,  0.0201,  ..., -0.0020, -0.0036,  0.0160]])), ('model.decoder.layers.0.self_attn.out_proj.bias', tensor([ 0.0195,  0.0254,  0.0588,  ...,  0.0776, -0.0562,  0.1057])), ('model.decoder.layers.0.self_attn_layer_norm.weight', tensor([0.6621, 0.6582, 0.8569,  ..., 0.3354, 0.3040, 0.2861])), ('model.decoder.layers.0.self_attn_layer_norm.bias', tensor([0.0337, 0.0765, 0.0483,  ..., 0.0493, 0.0312, 0.0363])), ('model.decoder.layers.0.encoder_attn.k_proj.weight', tensor([[ 0.0237, -0.0472, -0.0392,  ...,  0.0412,  0.0819, -0.0039],\n",
      "        [-0.0411,  0.0537,  0.0419,  ...,  0.0114, -0.1040,  0.0404],\n",
      "        [-0.1097,  0.0608,  0.0141,  ..., -0.0068, -0.0406, -0.0453],\n",
      "        ...,\n",
      "        [ 0.0176,  0.1121,  0.0232,  ...,  0.0023,  0.0341, -0.1055],\n",
      "        [-0.0114,  0.0332,  0.0285,  ..., -0.0486, -0.0189,  0.0422],\n",
      "        [ 0.0576, -0.0514,  0.0241,  ..., -0.0199, -0.0348, -0.0039]])), ('model.decoder.layers.0.encoder_attn.k_proj.bias', tensor([-0.2615,  0.1167, -0.0602,  ..., -0.3589,  0.1533,  0.1544])), ('model.decoder.layers.0.encoder_attn.v_proj.weight', tensor([[ 0.0342,  0.1056, -0.0375,  ...,  0.0312,  0.0592,  0.0328],\n",
      "        [-0.0172,  0.0225, -0.0968,  ..., -0.1266,  0.0399, -0.0540],\n",
      "        [ 0.0086,  0.0306,  0.1874,  ..., -0.0991, -0.0237,  0.0764],\n",
      "        ...,\n",
      "        [ 0.1265,  0.0562, -0.0228,  ...,  0.1165, -0.0486,  0.0134],\n",
      "        [ 0.1255,  0.0177, -0.0969,  ..., -0.0865, -0.0474, -0.1207],\n",
      "        [-0.0055,  0.1648, -0.0427,  ...,  0.0419, -0.1954,  0.0780]])), ('model.decoder.layers.0.encoder_attn.v_proj.bias', tensor([-0.0438, -0.0429,  0.1008,  ..., -0.1113, -0.0287, -0.0811])), ('model.decoder.layers.0.encoder_attn.q_proj.weight', tensor([[-0.0060,  0.1067,  0.0077,  ..., -0.0701, -0.0538,  0.0426],\n",
      "        [-0.0280, -0.0604,  0.0220,  ...,  0.0797,  0.0029, -0.0350],\n",
      "        [ 0.0451,  0.1091, -0.0186,  ..., -0.0313,  0.1597,  0.0523],\n",
      "        ...,\n",
      "        [ 0.1013,  0.0080, -0.1829,  ..., -0.0847,  0.0522, -0.0900],\n",
      "        [ 0.0978, -0.0154, -0.0279,  ...,  0.0037,  0.0491, -0.1027],\n",
      "        [ 0.0261,  0.1199,  0.0506,  ...,  0.0483,  0.0169, -0.0665]])), ('model.decoder.layers.0.encoder_attn.q_proj.bias', tensor([ 0.0918,  0.1081, -0.0681,  ...,  0.1530, -0.0709, -0.1768])), ('model.decoder.layers.0.encoder_attn.out_proj.weight', tensor([[-0.0085,  0.0075,  0.0027,  ..., -0.0378,  0.0050, -0.0735],\n",
      "        [ 0.0090,  0.0935,  0.0737,  ..., -0.0715, -0.0094,  0.0759],\n",
      "        [-0.0441,  0.1102,  0.0559,  ..., -0.0379, -0.0069,  0.0610],\n",
      "        ...,\n",
      "        [-0.0229,  0.1086,  0.0672,  ..., -0.0575,  0.0673,  0.0189],\n",
      "        [ 0.0623,  0.0409, -0.0999,  ..., -0.0527, -0.1014,  0.0045],\n",
      "        [ 0.0579, -0.0463, -0.0267,  ...,  0.0228, -0.0155,  0.0043]])), ('model.decoder.layers.0.encoder_attn.out_proj.bias', tensor([-0.0191, -0.0365,  0.0734,  ...,  0.0759, -0.0144,  0.1252])), ('model.decoder.layers.0.encoder_attn_layer_norm.weight', tensor([0.9453, 0.9429, 0.8735,  ..., 0.9160, 0.9614, 0.9067])), ('model.decoder.layers.0.encoder_attn_layer_norm.bias', tensor([-0.0349, -0.0727, -0.0472,  ..., -0.0572, -0.0343, -0.0751])), ('model.decoder.layers.0.fc1.weight', tensor([[ 0.0265, -0.0511, -0.0439,  ..., -0.0535, -0.0477,  0.0528],\n",
      "        [-0.0519,  0.0278, -0.1110,  ...,  0.0760,  0.0463, -0.1437],\n",
      "        [ 0.0716, -0.1112, -0.1459,  ...,  0.0220, -0.0909,  0.0653],\n",
      "        ...,\n",
      "        [ 0.0820,  0.1384,  0.1415,  ..., -0.0681,  0.1495, -0.0819],\n",
      "        [ 0.1160, -0.0803, -0.2111,  ..., -0.0130, -0.0774, -0.0189],\n",
      "        [ 0.1672, -0.1765, -0.0334,  ...,  0.1071,  0.0050, -0.1326]])), ('model.decoder.layers.0.fc1.bias', tensor([ 0.0551, -0.0403,  0.0269,  ..., -0.0837,  0.0421, -0.0320])), ('model.decoder.layers.0.fc2.weight', tensor([[ 0.0070,  0.0421, -0.0193,  ...,  0.0865,  0.0395, -0.1176],\n",
      "        [ 0.0867, -0.0462,  0.1218,  ..., -0.0995, -0.0442,  0.0738],\n",
      "        [-0.0163,  0.1393,  0.0247,  ...,  0.0934,  0.0512, -0.0343],\n",
      "        ...,\n",
      "        [ 0.0522, -0.1635, -0.0272,  ...,  0.0787, -0.0154, -0.0753],\n",
      "        [ 0.0623,  0.0332,  0.0449,  ...,  0.0289,  0.0775, -0.0364],\n",
      "        [-0.0101,  0.1043, -0.1458,  ..., -0.1144, -0.0363,  0.0742]])), ('model.decoder.layers.0.fc2.bias', tensor([-0.1212, -0.1192, -0.0242,  ..., -0.0781, -0.0556,  0.0465])), ('model.decoder.layers.0.final_layer_norm.weight', tensor([0.9727, 0.9951, 0.9902,  ..., 0.9380, 0.9668, 0.9399])), ('model.decoder.layers.0.final_layer_norm.bias', tensor([ 0.0189, -0.0003, -0.0134,  ..., -0.0126,  0.0524, -0.0305])), ('model.decoder.layers.1.self_attn.k_proj.weight', tensor([[ 0.0658,  0.0137,  0.1558,  ..., -0.0539, -0.0031,  0.1257],\n",
      "        [-0.0640,  0.0019,  0.1073,  ..., -0.0495, -0.1187,  0.1042],\n",
      "        [ 0.1077,  0.0075,  0.0540,  ..., -0.1261, -0.0953,  0.0054],\n",
      "        ...,\n",
      "        [-0.0445,  0.0246, -0.0106,  ...,  0.0522, -0.0161,  0.1136],\n",
      "        [ 0.0368,  0.0578,  0.0423,  ...,  0.0577,  0.0515, -0.0368],\n",
      "        [-0.1414, -0.0518, -0.0246,  ...,  0.0128,  0.0884, -0.1753]])), ('model.decoder.layers.1.self_attn.k_proj.bias', tensor([-0.0014, -0.7085,  0.7578,  ...,  0.4761, -0.7510,  0.5156])), ('model.decoder.layers.1.self_attn.v_proj.weight', tensor([[ 0.0640, -0.0595,  0.0584,  ..., -0.0301,  0.0520, -0.0493],\n",
      "        [-0.0632, -0.0405,  0.0491,  ..., -0.0340, -0.1344, -0.0085],\n",
      "        [-0.0232, -0.0614,  0.0628,  ..., -0.0356, -0.0350,  0.0154],\n",
      "        ...,\n",
      "        [ 0.0089, -0.1160,  0.1172,  ..., -0.2173,  0.1238, -0.1149],\n",
      "        [-0.1375,  0.0215, -0.0522,  ..., -0.1008,  0.0365, -0.1050],\n",
      "        [ 0.0448, -0.0862,  0.0554,  ...,  0.1182,  0.0487, -0.0187]])), ('model.decoder.layers.1.self_attn.v_proj.bias', tensor([-0.0138, -0.0024,  0.0232,  ..., -0.0058, -0.0362, -0.0105])), ('model.decoder.layers.1.self_attn.q_proj.weight', tensor([[-0.0431, -0.0268, -0.1415,  ...,  0.0108,  0.1030,  0.0761],\n",
      "        [ 0.1704,  0.0310,  0.1276,  ..., -0.0387, -0.0842, -0.0552],\n",
      "        [ 0.1990, -0.0290, -0.1871,  ...,  0.0857,  0.0973,  0.0571],\n",
      "        ...,\n",
      "        [-0.1243, -0.0435, -0.0263,  ..., -0.0053,  0.1062, -0.1265],\n",
      "        [-0.1550,  0.0648, -0.0323,  ..., -0.0357,  0.1138,  0.1147],\n",
      "        [-0.0486, -0.0043,  0.0336,  ..., -0.0298, -0.2393, -0.0880]])), ('model.decoder.layers.1.self_attn.q_proj.bias', tensor([ 0.0384,  0.0364, -0.0345,  ...,  0.0291,  0.0121,  0.0103])), ('model.decoder.layers.1.self_attn.out_proj.weight', tensor([[-0.0066, -0.1240, -0.0074,  ...,  0.0130,  0.0175,  0.0237],\n",
      "        [-0.0674, -0.1875,  0.0449,  ..., -0.0319,  0.0639, -0.0168],\n",
      "        [ 0.0576, -0.0472, -0.0279,  ..., -0.0691, -0.1137,  0.1244],\n",
      "        ...,\n",
      "        [ 0.0874,  0.0082, -0.1292,  ...,  0.1129, -0.0789, -0.0122],\n",
      "        [ 0.0559, -0.0823, -0.0209,  ...,  0.0712,  0.0080,  0.1077],\n",
      "        [-0.0270, -0.0399,  0.1316,  ..., -0.0196,  0.0295,  0.0079]])), ('model.decoder.layers.1.self_attn.out_proj.bias', tensor([ 0.0030,  0.0105,  0.0298,  ...,  0.0588, -0.0026,  0.0430])), ('model.decoder.layers.1.self_attn_layer_norm.weight', tensor([0.9395, 0.9580, 0.9634,  ..., 0.9248, 0.9482, 0.9028])), ('model.decoder.layers.1.self_attn_layer_norm.bias', tensor([ 0.0220,  0.0274,  0.0106,  ..., -0.0013,  0.0444, -0.0134])), ('model.decoder.layers.1.encoder_attn.k_proj.weight', tensor([[-0.0547, -0.0046,  0.0351,  ...,  0.0869,  0.0434, -0.0225],\n",
      "        [-0.0183, -0.0541, -0.0450,  ...,  0.0346,  0.0130, -0.0792],\n",
      "        [-0.0093, -0.0359, -0.0866,  ...,  0.0498,  0.0352,  0.0308],\n",
      "        ...,\n",
      "        [-0.0925, -0.0650,  0.0870,  ..., -0.0677,  0.0162,  0.0456],\n",
      "        [-0.1042,  0.0638,  0.0153,  ...,  0.0335, -0.0117,  0.0721],\n",
      "        [ 0.0884, -0.0022, -0.1285,  ..., -0.1224, -0.0282,  0.0357]])), ('model.decoder.layers.1.encoder_attn.k_proj.bias', tensor([-0.2668, -0.1351,  0.0676,  ..., -0.2795,  0.0698,  0.2433])), ('model.decoder.layers.1.encoder_attn.v_proj.weight', tensor([[ 0.0453,  0.0701,  0.1465,  ..., -0.0023,  0.0008, -0.0159],\n",
      "        [ 0.0526, -0.1270,  0.0374,  ..., -0.2386,  0.0282, -0.0166],\n",
      "        [-0.0704,  0.1145, -0.0098,  ..., -0.0398, -0.1411,  0.0015],\n",
      "        ...,\n",
      "        [ 0.0673,  0.0061, -0.0344,  ...,  0.0356,  0.1368,  0.0184],\n",
      "        [ 0.0377,  0.0097,  0.0800,  ...,  0.0754,  0.0217, -0.0288],\n",
      "        [-0.1083, -0.0229,  0.0258,  ...,  0.0374,  0.0677,  0.0159]])), ('model.decoder.layers.1.encoder_attn.v_proj.bias', tensor([-0.0310, -0.0053, -0.0002,  ..., -0.0063, -0.0195,  0.0006])), ('model.decoder.layers.1.encoder_attn.q_proj.weight', tensor([[-0.0107, -0.1378, -0.0382,  ...,  0.0016, -0.0966, -0.0055],\n",
      "        [-0.0515, -0.0196, -0.1290,  ..., -0.0209, -0.0578,  0.0949],\n",
      "        [-0.1357, -0.0212,  0.1285,  ..., -0.0648,  0.0224, -0.0066],\n",
      "        ...,\n",
      "        [ 0.0671, -0.0593,  0.0034,  ..., -0.0579,  0.0445, -0.0952],\n",
      "        [ 0.0444,  0.0798, -0.0031,  ...,  0.0206,  0.0157, -0.0495],\n",
      "        [ 0.0115, -0.0084,  0.0358,  ..., -0.0493, -0.0259, -0.0543]])), ('model.decoder.layers.1.encoder_attn.q_proj.bias', tensor([ 0.0842,  0.0052,  0.0588,  ..., -0.0193, -0.0054, -0.0160])), ('model.decoder.layers.1.encoder_attn.out_proj.weight', tensor([[-0.0574, -0.0395,  0.0453,  ..., -0.0228, -0.0005,  0.0886],\n",
      "        [-0.0377,  0.0698,  0.0314,  ..., -0.0673,  0.0032,  0.0632],\n",
      "        [-0.0726, -0.0036,  0.0242,  ...,  0.0773,  0.0754, -0.0518],\n",
      "        ...,\n",
      "        [-0.0691,  0.0414, -0.1033,  ...,  0.0531,  0.1542, -0.0355],\n",
      "        [-0.0483, -0.0524, -0.0430,  ...,  0.0694,  0.0912,  0.0668],\n",
      "        [ 0.0145,  0.0745, -0.0156,  ...,  0.1331, -0.0137,  0.0442]])), ('model.decoder.layers.1.encoder_attn.out_proj.bias', tensor([ 4.2999e-02,  5.9570e-02, -4.1544e-05,  ..., -1.1276e-02,\n",
      "         6.5155e-03, -4.9896e-02])), ('model.decoder.layers.1.encoder_attn_layer_norm.weight', tensor([0.9688, 0.9126, 0.8984,  ..., 0.9209, 0.9219, 0.9062])), ('model.decoder.layers.1.encoder_attn_layer_norm.bias', tensor([ 0.0096,  0.0257, -0.0100,  ..., -0.0219,  0.0356, -0.0004])), ('model.decoder.layers.1.fc1.weight', tensor([[ 3.4668e-02, -6.0150e-02,  1.3147e-01,  ..., -6.1615e-02,\n",
      "          6.0638e-02, -1.1554e-01],\n",
      "        [ 1.0675e-01,  1.3892e-01, -6.4941e-02,  ..., -3.4973e-02,\n",
      "         -2.0294e-02, -1.0864e-01],\n",
      "        [-2.4582e-02, -1.7078e-01,  2.5726e-02,  ...,  5.7526e-02,\n",
      "          5.3894e-02, -1.6876e-02],\n",
      "        ...,\n",
      "        [ 1.1792e-01,  6.3515e-03, -3.4253e-01,  ...,  3.1586e-02,\n",
      "         -8.8074e-02, -2.6321e-04],\n",
      "        [-6.4026e-02,  9.7229e-02, -1.5906e-01,  ..., -1.8542e-01,\n",
      "          1.8433e-02, -9.2285e-02],\n",
      "        [-4.9896e-02,  1.6516e-01,  5.7190e-02,  ...,  3.2104e-02,\n",
      "         -3.2776e-02, -5.3101e-02]])), ('model.decoder.layers.1.fc1.bias', tensor([-0.0585, -0.0326, -0.0437,  ..., -0.0746, -0.0337, -0.0074])), ('model.decoder.layers.1.fc2.weight', tensor([[ 0.0859,  0.0167,  0.0027,  ...,  0.0762,  0.0360,  0.0230],\n",
      "        [ 0.0844,  0.0935,  0.1317,  ...,  0.0393,  0.0353,  0.0191],\n",
      "        [ 0.0104, -0.0898,  0.0088,  ..., -0.0963, -0.0583, -0.0562],\n",
      "        ...,\n",
      "        [ 0.0323,  0.0246, -0.0977,  ...,  0.0024, -0.0480,  0.0956],\n",
      "        [-0.1354, -0.0197,  0.0545,  ...,  0.0519, -0.1171,  0.0428],\n",
      "        [-0.1503,  0.0646,  0.0820,  ..., -0.0084, -0.0452,  0.0864]])), ('model.decoder.layers.1.fc2.bias', tensor([-0.1240, -0.1116, -0.0324,  ..., -0.0482, -0.0426, -0.0128])), ('model.decoder.layers.1.final_layer_norm.weight', tensor([0.9839, 0.9824, 0.9902,  ..., 0.9741, 0.9624, 0.9409])), ('model.decoder.layers.1.final_layer_norm.bias', tensor([ 0.0275,  0.0312,  0.0129,  ..., -0.0210,  0.0584, -0.0214])), ('model.decoder.layers.2.self_attn.k_proj.weight', tensor([[-0.0505, -0.0604,  0.1780,  ...,  0.0571, -0.0449, -0.0190],\n",
      "        [ 0.0413,  0.0961,  0.0137,  ...,  0.0699, -0.0218,  0.0573],\n",
      "        [ 0.0492, -0.0178, -0.0190,  ..., -0.0825,  0.0950,  0.1864],\n",
      "        ...,\n",
      "        [ 0.0701,  0.1406,  0.0816,  ...,  0.1168,  0.2216,  0.1069],\n",
      "        [-0.0833,  0.0252,  0.0728,  ..., -0.1136, -0.0066,  0.0704],\n",
      "        [-0.0586, -0.0621, -0.0082,  ...,  0.0914, -0.0497, -0.1570]])), ('model.decoder.layers.2.self_attn.k_proj.bias', tensor([-0.0879,  0.3384, -0.0207,  ...,  0.0334, -0.0283, -0.0624])), ('model.decoder.layers.2.self_attn.v_proj.weight', tensor([[-0.0207, -0.1207, -0.0385,  ..., -0.1176, -0.0531,  0.0395],\n",
      "        [-0.0942,  0.0064,  0.2014,  ...,  0.0454,  0.0013,  0.2910],\n",
      "        [ 0.0323, -0.0977, -0.0571,  ...,  0.0335,  0.0667,  0.1250],\n",
      "        ...,\n",
      "        [-0.0365, -0.1248, -0.0659,  ..., -0.0491,  0.0245, -0.0040],\n",
      "        [-0.2754, -0.0710, -0.0831,  ...,  0.2378, -0.0217, -0.0692],\n",
      "        [-0.1503, -0.0089,  0.0410,  ...,  0.0718,  0.0469, -0.0829]])), ('model.decoder.layers.2.self_attn.v_proj.bias', tensor([ 0.0359, -0.0082, -0.0071,  ...,  0.0041, -0.0007, -0.0241])), ('model.decoder.layers.2.self_attn.q_proj.weight', tensor([[ 0.0178, -0.0066,  0.1232,  ..., -0.0901,  0.1736, -0.0714],\n",
      "        [ 0.0886, -0.3564,  0.1796,  ..., -0.1323,  0.0145,  0.0226],\n",
      "        [ 0.0869, -0.1311,  0.1348,  ...,  0.0681, -0.1743,  0.0952],\n",
      "        ...,\n",
      "        [ 0.1638,  0.0925,  0.1174,  ...,  0.1648, -0.2751, -0.0351],\n",
      "        [ 0.0613, -0.0438,  0.1172,  ..., -0.0164,  0.0481, -0.0189],\n",
      "        [ 0.2466,  0.2661,  0.1543,  ..., -0.0739,  0.2585, -0.1104]])), ('model.decoder.layers.2.self_attn.q_proj.bias', tensor([-0.0056,  0.0203,  0.0063,  ...,  0.0133, -0.0052, -0.0107])), ('model.decoder.layers.2.self_attn.out_proj.weight', tensor([[-0.0032, -0.0740,  0.0231,  ...,  0.0218, -0.0123,  0.0027],\n",
      "        [-0.1331,  0.0390,  0.0407,  ..., -0.1675, -0.0197, -0.0767],\n",
      "        [ 0.0387, -0.0259, -0.0529,  ..., -0.0294, -0.0075,  0.1251],\n",
      "        ...,\n",
      "        [-0.0510,  0.0412,  0.0704,  ..., -0.1124,  0.0354,  0.1252],\n",
      "        [ 0.0814,  0.0601,  0.0422,  ...,  0.0347, -0.0345, -0.0140],\n",
      "        [ 0.0069,  0.1394, -0.0816,  ...,  0.0162,  0.0277,  0.0135]])), ('model.decoder.layers.2.self_attn.out_proj.bias', tensor([ 0.0671,  0.0434,  0.0201,  ..., -0.0056,  0.0233, -0.0598])), ('model.decoder.layers.2.self_attn_layer_norm.weight', tensor([0.9839, 0.9707, 0.9907,  ..., 0.9736, 0.9917, 0.9478])), ('model.decoder.layers.2.self_attn_layer_norm.bias', tensor([ 1.7883e-02,  1.6617e-02,  1.4809e-02,  ..., -1.2577e-05,\n",
      "         4.5410e-02,  4.5509e-03])), ('model.decoder.layers.2.encoder_attn.k_proj.weight', tensor([[ 0.0558, -0.0396, -0.0997,  ..., -0.0039,  0.0525, -0.0451],\n",
      "        [ 0.0273, -0.1348, -0.0333,  ..., -0.0959, -0.0339,  0.0829],\n",
      "        [-0.0789, -0.1163,  0.1069,  ..., -0.0400,  0.1313,  0.0725],\n",
      "        ...,\n",
      "        [ 0.0364, -0.0713, -0.0695,  ..., -0.0450,  0.0600, -0.0675],\n",
      "        [ 0.1037, -0.0692,  0.0147,  ...,  0.0923, -0.0746,  0.0999],\n",
      "        [ 0.1731,  0.0196,  0.0468,  ..., -0.1273,  0.0629, -0.0060]])), ('model.decoder.layers.2.encoder_attn.k_proj.bias', tensor([ 0.3591, -0.1003,  0.1299,  ..., -0.0033,  0.0542, -0.0901])), ('model.decoder.layers.2.encoder_attn.v_proj.weight', tensor([[ 0.0537,  0.0516,  0.0286,  ...,  0.0669, -0.0040,  0.0283],\n",
      "        [-0.0732, -0.1360, -0.1124,  ..., -0.0073, -0.0775,  0.0053],\n",
      "        [ 0.0659,  0.0125, -0.2480,  ..., -0.0048, -0.0299,  0.0201],\n",
      "        ...,\n",
      "        [-0.0494,  0.1277, -0.1908,  ..., -0.0170, -0.0938, -0.1025],\n",
      "        [-0.0363,  0.1118,  0.0860,  ...,  0.1120, -0.0890,  0.0018],\n",
      "        [ 0.0836,  0.2178, -0.0050,  ..., -0.1322, -0.0574,  0.0801]])), ('model.decoder.layers.2.encoder_attn.v_proj.bias', tensor([ 0.0104, -0.0011, -0.0359,  ...,  0.0140,  0.0363,  0.0296])), ('model.decoder.layers.2.encoder_attn.q_proj.weight', tensor([[-0.0872,  0.0089, -0.0074,  ..., -0.0274,  0.0192, -0.0388],\n",
      "        [-0.1577, -0.0691, -0.0707,  ...,  0.0075,  0.0123, -0.0818],\n",
      "        [-0.1084,  0.0413, -0.0580,  ...,  0.0480,  0.0425, -0.0329],\n",
      "        ...,\n",
      "        [-0.0527,  0.0797,  0.0980,  ...,  0.0541, -0.0768, -0.0517],\n",
      "        [ 0.0087,  0.0116,  0.0339,  ...,  0.0462,  0.0230,  0.0746],\n",
      "        [ 0.0265,  0.0215,  0.0036,  ..., -0.2225,  0.0161,  0.0686]])), ('model.decoder.layers.2.encoder_attn.q_proj.bias', tensor([-0.0112,  0.0030, -0.0094,  ..., -0.0076, -0.0092, -0.0698])), ('model.decoder.layers.2.encoder_attn.out_proj.weight', tensor([[ 0.0403, -0.0240,  0.0201,  ...,  0.0300,  0.0855, -0.0436],\n",
      "        [ 0.1121,  0.0231, -0.0040,  ...,  0.0184,  0.0737, -0.0944],\n",
      "        [ 0.0181, -0.0997,  0.0718,  ..., -0.1262, -0.1214, -0.0452],\n",
      "        ...,\n",
      "        [ 0.0356,  0.0366,  0.0499,  ...,  0.0419,  0.0439,  0.0314],\n",
      "        [-0.0587,  0.1107,  0.0019,  ..., -0.0405, -0.0637, -0.1023],\n",
      "        [-0.0473, -0.0702, -0.0184,  ..., -0.0592,  0.0202, -0.0931]])), ('model.decoder.layers.2.encoder_attn.out_proj.bias', tensor([ 0.0876,  0.0832,  0.0145,  ..., -0.0225,  0.0349, -0.0126])), ('model.decoder.layers.2.encoder_attn_layer_norm.weight', tensor([0.9849, 0.9189, 0.8369,  ..., 0.8838, 0.9390, 0.8828])), ('model.decoder.layers.2.encoder_attn_layer_norm.bias', tensor([ 0.0254,  0.0205,  0.0016,  ..., -0.0204,  0.0431, -0.0004])), ('model.decoder.layers.2.fc1.weight', tensor([[ 0.1752,  0.1364,  0.1648,  ...,  0.0822,  0.0662, -0.0356],\n",
      "        [-0.2615,  0.1376, -0.0718,  ...,  0.1617, -0.1051,  0.1432],\n",
      "        [-0.0867,  0.0024,  0.1879,  ..., -0.0108,  0.1224,  0.0410],\n",
      "        ...,\n",
      "        [-0.1053, -0.0575,  0.1112,  ...,  0.0191,  0.1134, -0.1646],\n",
      "        [-0.1425, -0.0595,  0.0013,  ...,  0.1247, -0.0166,  0.0795],\n",
      "        [ 0.0484,  0.0943, -0.1560,  ..., -0.0600, -0.0270,  0.1168]])), ('model.decoder.layers.2.fc1.bias', tensor([-0.0753, -0.0708, -0.0108,  ...,  0.0061,  0.0109, -0.0681])), ('model.decoder.layers.2.fc2.weight', tensor([[-0.0513, -0.0304,  0.0806,  ...,  0.1089,  0.1111,  0.0185],\n",
      "        [ 0.0176,  0.0331, -0.0380,  ...,  0.0765, -0.0172,  0.0547],\n",
      "        [ 0.0040,  0.0286, -0.0994,  ..., -0.0883,  0.0676, -0.0480],\n",
      "        ...,\n",
      "        [-0.0506, -0.0814,  0.0792,  ..., -0.0497, -0.0449, -0.0379],\n",
      "        [ 0.0809, -0.0018, -0.0311,  ..., -0.1178,  0.0401, -0.0158],\n",
      "        [-0.1572,  0.0816,  0.0397,  ...,  0.1302, -0.0411,  0.1118]])), ('model.decoder.layers.2.fc2.bias', tensor([-0.0728, -0.0604, -0.0215,  ..., -0.0622, -0.0247, -0.0003])), ('model.decoder.layers.2.final_layer_norm.weight', tensor([0.9683, 0.9946, 0.9976,  ..., 0.9883, 0.9644, 0.9653])), ('model.decoder.layers.2.final_layer_norm.bias', tensor([ 0.0126,  0.0242,  0.0122,  ..., -0.0154,  0.0421, -0.0019])), ('model.decoder.layers.3.self_attn.k_proj.weight', tensor([[-0.0325, -0.0724, -0.0953,  ..., -0.0622,  0.0018, -0.1183],\n",
      "        [-0.1364, -0.1255,  0.0957,  ...,  0.0238, -0.0504,  0.0970],\n",
      "        [-0.1466, -0.0804,  0.0126,  ..., -0.0652,  0.0661,  0.1287],\n",
      "        ...,\n",
      "        [ 0.0984, -0.0325,  0.0580,  ..., -0.0264,  0.0149, -0.0224],\n",
      "        [ 0.0456, -0.0674,  0.0077,  ...,  0.0607, -0.0068, -0.1422],\n",
      "        [-0.1458, -0.0828,  0.0690,  ...,  0.0285,  0.0469,  0.0558]])), ('model.decoder.layers.3.self_attn.k_proj.bias', tensor([-0.1713,  0.0254,  0.1830,  ..., -0.2510, -0.0788, -0.0329])), ('model.decoder.layers.3.self_attn.v_proj.weight', tensor([[-0.0031, -0.0759,  0.0630,  ..., -0.0801, -0.0272, -0.1192],\n",
      "        [ 0.0169, -0.0367,  0.0119,  ...,  0.0181,  0.0391,  0.0195],\n",
      "        [-0.0359, -0.0353, -0.1652,  ..., -0.0950, -0.0469, -0.0522],\n",
      "        ...,\n",
      "        [ 0.0428, -0.0086,  0.0161,  ...,  0.0902, -0.0050, -0.0060],\n",
      "        [ 0.1648, -0.1892, -0.0332,  ..., -0.1543, -0.0077, -0.0709],\n",
      "        [ 0.1375,  0.0040, -0.0917,  ..., -0.0024,  0.0723,  0.0213]])), ('model.decoder.layers.3.self_attn.v_proj.bias', tensor([-0.0107, -0.0301, -0.0077,  ...,  0.0341,  0.0276, -0.0185])), ('model.decoder.layers.3.self_attn.q_proj.weight', tensor([[ 0.0740, -0.1107,  0.0903,  ..., -0.1643, -0.0685,  0.1332],\n",
      "        [ 0.0942,  0.1335,  0.1553,  ..., -0.0934,  0.1399, -0.1710],\n",
      "        [ 0.0415, -0.0698, -0.0672,  ..., -0.0522,  0.1559,  0.1243],\n",
      "        ...,\n",
      "        [ 0.0609,  0.1609,  0.1621,  ..., -0.1997, -0.0304,  0.1039],\n",
      "        [-0.1564, -0.2297,  0.0279,  ...,  0.0160,  0.1116, -0.0187],\n",
      "        [-0.1870, -0.0023,  0.0533,  ...,  0.1157, -0.0312, -0.0718]])), ('model.decoder.layers.3.self_attn.q_proj.bias', tensor([-0.0152,  0.0182,  0.0058,  ..., -0.0409,  0.0207,  0.0064])), ('model.decoder.layers.3.self_attn.out_proj.weight', tensor([[-1.7303e-02, -1.4679e-02, -2.1698e-02,  ..., -8.2092e-02,\n",
      "         -9.0576e-02, -9.7900e-02],\n",
      "        [-7.5012e-02, -1.0437e-01, -3.0991e-02,  ...,  3.5431e-02,\n",
      "          1.4490e-01,  4.0375e-02],\n",
      "        [-3.7445e-02,  2.5368e-03,  3.5703e-05,  ...,  1.9135e-02,\n",
      "         -9.1095e-03,  1.0944e-01],\n",
      "        ...,\n",
      "        [-2.4673e-02,  4.5776e-02, -2.2919e-02,  ..., -1.5955e-01,\n",
      "          9.0820e-02,  1.2732e-01],\n",
      "        [ 1.2103e-01,  5.2124e-02, -4.0009e-02,  ..., -4.8752e-03,\n",
      "         -9.4482e-02,  9.2651e-02],\n",
      "        [-1.0608e-01, -1.0117e-02, -8.3208e-04,  ..., -3.9917e-02,\n",
      "          7.6355e-02, -4.0092e-03]])), ('model.decoder.layers.3.self_attn.out_proj.bias', tensor([ 0.0393,  0.0549,  0.0605,  ..., -0.0289,  0.0244, -0.0362])), ('model.decoder.layers.3.self_attn_layer_norm.weight', tensor([0.9678, 0.9658, 0.9951,  ..., 0.9668, 0.9775, 0.9751])), ('model.decoder.layers.3.self_attn_layer_norm.bias', tensor([ 0.0128,  0.0056,  0.0140,  ..., -0.0010,  0.0389,  0.0162])), ('model.decoder.layers.3.encoder_attn.k_proj.weight', tensor([[ 0.0710, -0.0582, -0.1216,  ..., -0.1005,  0.0967,  0.0915],\n",
      "        [ 0.0779, -0.0219, -0.0660,  ...,  0.0270, -0.0315,  0.0786],\n",
      "        [-0.0774, -0.0079,  0.0042,  ...,  0.0359,  0.0174,  0.0871],\n",
      "        ...,\n",
      "        [ 0.0478,  0.0173,  0.1252,  ...,  0.0998, -0.0237, -0.0265],\n",
      "        [ 0.1530, -0.0086,  0.0159,  ..., -0.0263, -0.0045, -0.0486],\n",
      "        [-0.1582, -0.0670, -0.0285,  ..., -0.0586,  0.0784,  0.0569]])), ('model.decoder.layers.3.encoder_attn.k_proj.bias', tensor([-0.2302,  0.0732, -0.0722,  ...,  0.3237, -0.1298,  0.0041])), ('model.decoder.layers.3.encoder_attn.v_proj.weight', tensor([[-0.1030, -0.0007, -0.1212,  ..., -0.0928, -0.0316, -0.0098],\n",
      "        [-0.0503,  0.0470, -0.0040,  ..., -0.0533,  0.0894,  0.0015],\n",
      "        [ 0.0825, -0.1215, -0.1675,  ...,  0.0410, -0.0036,  0.0414],\n",
      "        ...,\n",
      "        [ 0.0048,  0.0084,  0.2157,  ...,  0.0930, -0.0117, -0.1376],\n",
      "        [-0.0291, -0.0060, -0.0552,  ..., -0.1770, -0.1045,  0.1222],\n",
      "        [ 0.0423, -0.0461, -0.1037,  ..., -0.0685, -0.0370,  0.0795]])), ('model.decoder.layers.3.encoder_attn.v_proj.bias', tensor([-0.0497, -0.0240,  0.0234,  ..., -0.0254, -0.0070, -0.0003])), ('model.decoder.layers.3.encoder_attn.q_proj.weight', tensor([[ 0.1356,  0.0191,  0.1242,  ..., -0.0394,  0.0908,  0.1578],\n",
      "        [ 0.0948,  0.0530,  0.1259,  ...,  0.0336,  0.0239, -0.0531],\n",
      "        [-0.1245, -0.0602,  0.1036,  ...,  0.0979, -0.0773, -0.0212],\n",
      "        ...,\n",
      "        [ 0.0898,  0.0217,  0.0580,  ..., -0.0705, -0.0018, -0.1003],\n",
      "        [ 0.1313,  0.0606,  0.2349,  ...,  0.0230,  0.0285,  0.1661],\n",
      "        [ 0.1696,  0.0017, -0.0048,  ..., -0.0887, -0.0045,  0.1368]])), ('model.decoder.layers.3.encoder_attn.q_proj.bias', tensor([ 0.0263, -0.0505, -0.0086,  ...,  0.0214, -0.0023, -0.0307])), ('model.decoder.layers.3.encoder_attn.out_proj.weight', tensor([[-0.0710,  0.0457,  0.0874,  ...,  0.0743, -0.1562,  0.0481],\n",
      "        [ 0.0218,  0.0564, -0.0660,  ..., -0.0159, -0.1420,  0.0209],\n",
      "        [-0.1406,  0.0663,  0.0951,  ...,  0.0656, -0.1835,  0.1000],\n",
      "        ...,\n",
      "        [-0.0112, -0.0124,  0.0518,  ...,  0.0559, -0.0241,  0.0225],\n",
      "        [-0.0340,  0.1343, -0.0138,  ...,  0.0050,  0.0601, -0.1578],\n",
      "        [-0.0367,  0.0493, -0.0228,  ...,  0.0110, -0.0229,  0.1042]])), ('model.decoder.layers.3.encoder_attn.out_proj.bias', tensor([ 0.0940,  0.0368,  0.0245,  ..., -0.0054,  0.0318, -0.0234])), ('model.decoder.layers.3.encoder_attn_layer_norm.weight', tensor([0.9814, 0.9844, 0.8691,  ..., 0.9077, 0.9292, 0.8506])), ('model.decoder.layers.3.encoder_attn_layer_norm.bias', tensor([ 0.0183,  0.0132,  0.0036,  ..., -0.0200,  0.0284,  0.0145])), ('model.decoder.layers.3.fc1.weight', tensor([[ 3.5736e-02,  8.2092e-03,  6.0913e-02,  ...,  8.2031e-02,\n",
      "          1.1711e-02,  8.5327e-02],\n",
      "        [-5.3497e-02,  7.6294e-02,  5.8960e-02,  ..., -2.1577e-04,\n",
      "         -9.8450e-02,  1.2457e-01],\n",
      "        [ 1.0382e-01, -2.5000e-01, -4.1840e-02,  ...,  1.1163e-01,\n",
      "         -2.1838e-01,  2.0471e-01],\n",
      "        ...,\n",
      "        [-5.3894e-02, -8.2642e-02, -3.4547e-04,  ...,  8.3496e-02,\n",
      "         -1.6589e-01, -1.6037e-02],\n",
      "        [ 1.5649e-01, -4.8561e-03, -9.7473e-02,  ...,  1.7075e-02,\n",
      "         -5.0293e-02,  1.2231e-01],\n",
      "        [ 7.0076e-03,  6.5552e-02,  2.8992e-04,  ...,  3.8971e-02,\n",
      "          7.0007e-02,  1.7563e-02]])), ('model.decoder.layers.3.fc1.bias', tensor([-0.0155, -0.0215,  0.0102,  ..., -0.0696, -0.0261, -0.0357])), ('model.decoder.layers.3.fc2.weight', tensor([[-0.1729, -0.1368, -0.0466,  ..., -0.0368, -0.0907,  0.0261],\n",
      "        [ 0.1159, -0.0869,  0.1608,  ...,  0.1328,  0.0079, -0.0302],\n",
      "        [ 0.1731,  0.0765, -0.0249,  ..., -0.1152, -0.0928, -0.0182],\n",
      "        ...,\n",
      "        [-0.0027, -0.0993, -0.0112,  ...,  0.0657, -0.0110, -0.0344],\n",
      "        [-0.0797, -0.0883, -0.0084,  ..., -0.0086, -0.0442, -0.0728],\n",
      "        [ 0.0163,  0.0443, -0.2147,  ..., -0.0229,  0.0053, -0.1063]])), ('model.decoder.layers.3.fc2.bias', tensor([-0.0617, -0.0828, -0.0044,  ..., -0.0246, -0.0031, -0.0078])), ('model.decoder.layers.3.final_layer_norm.weight', tensor([0.9648, 0.9839, 0.9858,  ..., 0.9863, 0.9766, 1.0020])), ('model.decoder.layers.3.final_layer_norm.bias', tensor([ 0.0065,  0.0138,  0.0101,  ..., -0.0110,  0.0330,  0.0059])), ('model.decoder.layers.4.self_attn.k_proj.weight', tensor([[-0.0437,  0.1418,  0.0049,  ...,  0.0358, -0.0906,  0.0452],\n",
      "        [ 0.0955, -0.1848,  0.1009,  ...,  0.0856, -0.0840,  0.0119],\n",
      "        [-0.0048,  0.0540,  0.0430,  ...,  0.2256, -0.1813,  0.1459],\n",
      "        ...,\n",
      "        [-0.1830, -0.0137,  0.0230,  ..., -0.0955,  0.0704, -0.0427],\n",
      "        [-0.0656, -0.0109,  0.1016,  ...,  0.0169, -0.0338,  0.0443],\n",
      "        [-0.1362,  0.0189, -0.0656,  ..., -0.1510, -0.0766,  0.0101]])), ('model.decoder.layers.4.self_attn.k_proj.bias', tensor([ 0.2966, -0.3657,  0.0850,  ..., -0.1017, -0.0119, -0.0157])), ('model.decoder.layers.4.self_attn.v_proj.weight', tensor([[ 0.0742, -0.0066, -0.0750,  ..., -0.0180,  0.1797, -0.0512],\n",
      "        [ 0.0615, -0.0837,  0.0469,  ...,  0.0253, -0.0814,  0.0845],\n",
      "        [ 0.0861,  0.0045,  0.0344,  ...,  0.0833,  0.1495, -0.1508],\n",
      "        ...,\n",
      "        [ 0.0141,  0.2139, -0.1414,  ..., -0.0093,  0.0517, -0.0260],\n",
      "        [ 0.0345, -0.2086,  0.1675,  ...,  0.0212,  0.0930, -0.0045],\n",
      "        [-0.0108, -0.0204,  0.0157,  ...,  0.0113,  0.0858,  0.0764]])), ('model.decoder.layers.4.self_attn.v_proj.bias', tensor([ 0.0374, -0.0064, -0.0150,  ..., -0.0014, -0.0092,  0.0008])), ('model.decoder.layers.4.self_attn.q_proj.weight', tensor([[ 0.0667, -0.0618, -0.0822,  ...,  0.0512,  0.1213,  0.0599],\n",
      "        [ 0.0573,  0.1121,  0.1710,  ..., -0.0924, -0.0952,  0.1010],\n",
      "        [ 0.0311, -0.0051, -0.0063,  ..., -0.2323,  0.0613, -0.0366],\n",
      "        ...,\n",
      "        [ 0.1561, -0.1512, -0.0891,  ...,  0.0637, -0.0239, -0.1089],\n",
      "        [-0.1633, -0.1660,  0.0292,  ..., -0.1439, -0.0768,  0.1080],\n",
      "        [-0.0787, -0.1418,  0.2002,  ..., -0.1200, -0.2133,  0.0684]])), ('model.decoder.layers.4.self_attn.q_proj.bias', tensor([ 0.0220, -0.0531,  0.0031,  ...,  0.0136,  0.0079, -0.0128])), ('model.decoder.layers.4.self_attn.out_proj.weight', tensor([[ 0.0645, -0.0244,  0.0408,  ...,  0.0373, -0.0029, -0.0374],\n",
      "        [-0.0609,  0.0148,  0.0823,  ..., -0.0959,  0.0609, -0.0093],\n",
      "        [-0.0022,  0.0692,  0.0895,  ...,  0.0028, -0.0748, -0.0597],\n",
      "        ...,\n",
      "        [-0.0251, -0.0115, -0.1758,  ..., -0.1420,  0.0458, -0.0104],\n",
      "        [ 0.1046,  0.1008,  0.0398,  ..., -0.0784, -0.0327, -0.0538],\n",
      "        [-0.1056, -0.0679, -0.1302,  ..., -0.1360, -0.0720, -0.1219]])), ('model.decoder.layers.4.self_attn.out_proj.bias', tensor([ 0.0180,  0.0323,  0.0135,  ..., -0.0424,  0.0050, -0.0271])), ('model.decoder.layers.4.self_attn_layer_norm.weight', tensor([0.9810, 0.9736, 0.9844,  ..., 0.9414, 0.9902, 0.9873])), ('model.decoder.layers.4.self_attn_layer_norm.bias', tensor([ 0.0060, -0.0024,  0.0177,  ...,  0.0024,  0.0343,  0.0219])), ('model.decoder.layers.4.encoder_attn.k_proj.weight', tensor([[ 0.0313,  0.0626,  0.0111,  ...,  0.0570, -0.0372,  0.0307],\n",
      "        [ 0.0446, -0.0288,  0.0111,  ...,  0.1265, -0.0079,  0.0117],\n",
      "        [-0.0318,  0.1493,  0.0493,  ...,  0.0139, -0.0901, -0.0069],\n",
      "        ...,\n",
      "        [ 0.0213, -0.0927, -0.0207,  ...,  0.0848, -0.0400,  0.0721],\n",
      "        [-0.1884,  0.0219,  0.0254,  ..., -0.1265,  0.0115, -0.1915],\n",
      "        [-0.0672, -0.1279, -0.1086,  ...,  0.0403, -0.1522,  0.0291]])), ('model.decoder.layers.4.encoder_attn.k_proj.bias', tensor([-0.0245, -0.0010,  0.0942,  ..., -0.3296, -0.0045,  0.0304])), ('model.decoder.layers.4.encoder_attn.v_proj.weight', tensor([[-0.0462, -0.0688,  0.1274,  ...,  0.0086,  0.0218,  0.0940],\n",
      "        [ 0.0135,  0.0548, -0.0382,  ..., -0.0208,  0.0480,  0.0591],\n",
      "        [ 0.0751,  0.0941, -0.1525,  ..., -0.0444,  0.2983, -0.1389],\n",
      "        ...,\n",
      "        [ 0.0333, -0.0260, -0.1277,  ..., -0.0989,  0.0577,  0.0851],\n",
      "        [-0.0302, -0.0085,  0.0988,  ...,  0.0047, -0.0482, -0.0342],\n",
      "        [ 0.0039, -0.1431,  0.0365,  ..., -0.0260,  0.0250, -0.0105]])), ('model.decoder.layers.4.encoder_attn.v_proj.bias', tensor([ 0.0040, -0.0098,  0.0111,  ..., -0.0015, -0.0063,  0.0071])), ('model.decoder.layers.4.encoder_attn.q_proj.weight', tensor([[-0.0154,  0.0786,  0.0129,  ...,  0.1176,  0.0742, -0.1411],\n",
      "        [-0.0303,  0.1017, -0.0352,  ..., -0.0581,  0.0878,  0.0778],\n",
      "        [-0.0692,  0.0955,  0.0168,  ..., -0.0532, -0.0390,  0.0731],\n",
      "        ...,\n",
      "        [-0.0082,  0.1174,  0.0724,  ..., -0.0181,  0.0778, -0.0591],\n",
      "        [-0.0741, -0.0146,  0.0134,  ..., -0.0232, -0.0212, -0.1451],\n",
      "        [ 0.0933, -0.1167,  0.0624,  ..., -0.0898, -0.0008, -0.0605]])), ('model.decoder.layers.4.encoder_attn.q_proj.bias', tensor([-0.0104,  0.0341, -0.0023,  ...,  0.0045,  0.0335,  0.0434])), ('model.decoder.layers.4.encoder_attn.out_proj.weight', tensor([[-0.0166,  0.0562,  0.0023,  ...,  0.0132, -0.0667, -0.0281],\n",
      "        [ 0.0853, -0.0527, -0.0505,  ...,  0.0593, -0.0303,  0.0382],\n",
      "        [ 0.0181, -0.1224, -0.0289,  ...,  0.0162, -0.1033, -0.0509],\n",
      "        ...,\n",
      "        [-0.0168, -0.1650, -0.0453,  ..., -0.0169,  0.1309,  0.0284],\n",
      "        [ 0.0249,  0.0224, -0.0662,  ...,  0.1132,  0.0980, -0.0500],\n",
      "        [ 0.0676, -0.0510, -0.1290,  ...,  0.0822,  0.0320, -0.0360]])), ('model.decoder.layers.4.encoder_attn.out_proj.bias', tensor([ 0.0681,  0.0690,  0.0067,  ..., -0.0359,  0.0337, -0.0482])), ('model.decoder.layers.4.encoder_attn_layer_norm.weight', tensor([0.9331, 0.9106, 0.8530,  ..., 0.8633, 0.9106, 0.8794])), ('model.decoder.layers.4.encoder_attn_layer_norm.bias', tensor([ 0.0131,  0.0175,  0.0072,  ..., -0.0139,  0.0241,  0.0130])), ('model.decoder.layers.4.fc1.weight', tensor([[-0.1182,  0.0400, -0.1996,  ..., -0.1888,  0.1244, -0.0388],\n",
      "        [-0.0044,  0.0397, -0.0410,  ...,  0.1305, -0.0970,  0.1257],\n",
      "        [ 0.0508, -0.0949,  0.0941,  ..., -0.0201, -0.0421, -0.1593],\n",
      "        ...,\n",
      "        [-0.0546,  0.1943, -0.0147,  ..., -0.0532,  0.0193, -0.0257],\n",
      "        [-0.0342,  0.0673,  0.1047,  ...,  0.1632, -0.1146,  0.0099],\n",
      "        [ 0.2489, -0.1426,  0.0778,  ...,  0.0501,  0.0756,  0.1357]])), ('model.decoder.layers.4.fc1.bias', tensor([-0.0339, -0.0529,  0.0283,  ..., -0.0059, -0.0391, -0.0487])), ('model.decoder.layers.4.fc2.weight', tensor([[-0.0694, -0.0710, -0.0245,  ...,  0.0712,  0.0591, -0.0290],\n",
      "        [ 0.0712,  0.0210,  0.0566,  ..., -0.1014, -0.0288,  0.0588],\n",
      "        [-0.1461, -0.0262,  0.0277,  ...,  0.0284,  0.1229, -0.0504],\n",
      "        ...,\n",
      "        [-0.0887, -0.2096,  0.0693,  ...,  0.0983, -0.0183, -0.0889],\n",
      "        [ 0.1499,  0.0572, -0.0114,  ..., -0.0662, -0.0516,  0.0338],\n",
      "        [-0.1038, -0.1012,  0.1045,  ...,  0.0490, -0.0106,  0.1014]])), ('model.decoder.layers.4.fc2.bias', tensor([-0.0285, -0.0835, -0.0043,  ..., -0.0050, -0.0044,  0.0058])), ('model.decoder.layers.4.final_layer_norm.weight', tensor([0.9565, 0.9819, 0.9609,  ..., 0.9771, 0.9546, 0.9785])), ('model.decoder.layers.4.final_layer_norm.bias', tensor([ 0.0074,  0.0125,  0.0104,  ..., -0.0039,  0.0209,  0.0121])), ('model.decoder.layers.5.self_attn.k_proj.weight', tensor([[-0.0064, -0.1274,  0.0425,  ...,  0.0159,  0.1437, -0.0784],\n",
      "        [-0.0352,  0.0724, -0.1422,  ..., -0.1569, -0.0412, -0.0634],\n",
      "        [ 0.1088,  0.2048, -0.1636,  ..., -0.0075,  0.0035, -0.0009],\n",
      "        ...,\n",
      "        [ 0.1368, -0.0883, -0.0434,  ...,  0.0410,  0.1422, -0.1175],\n",
      "        [ 0.0348,  0.0629,  0.0878,  ..., -0.0441,  0.0792, -0.0051],\n",
      "        [ 0.0028,  0.1089,  0.2661,  ..., -0.0836,  0.0350,  0.0856]])), ('model.decoder.layers.5.self_attn.k_proj.bias', tensor([ 0.0947, -0.0400, -0.0791,  ...,  0.0625,  0.1453,  0.0498])), ('model.decoder.layers.5.self_attn.v_proj.weight', tensor([[-0.0296, -0.0217, -0.0535,  ...,  0.0130, -0.0386,  0.1804],\n",
      "        [ 0.0388, -0.0807, -0.0011,  ..., -0.0513,  0.0132, -0.1974],\n",
      "        [-0.1602,  0.1187, -0.0398,  ..., -0.1059, -0.1077,  0.1793],\n",
      "        ...,\n",
      "        [ 0.0227, -0.0346, -0.1119,  ...,  0.1991,  0.0302,  0.0244],\n",
      "        [ 0.0259, -0.0097,  0.1178,  ..., -0.0157, -0.0455,  0.0149],\n",
      "        [-0.1099,  0.1492,  0.0519,  ..., -0.0532, -0.0082,  0.0221]])), ('model.decoder.layers.5.self_attn.v_proj.bias', tensor([ 1.3618e-03, -3.5286e-05, -2.3224e-02,  ...,  4.7974e-02,\n",
      "        -1.4763e-03,  1.0895e-02])), ('model.decoder.layers.5.self_attn.q_proj.weight', tensor([[ 3.2745e-02, -7.1899e-02, -1.2978e-02,  ..., -1.2170e-01,\n",
      "          4.6417e-02, -6.0089e-02],\n",
      "        [ 5.8929e-02, -1.2115e-01, -1.0883e-01,  ...,  1.0974e-01,\n",
      "          3.9703e-02,  1.5503e-01],\n",
      "        [ 3.1219e-02, -1.0681e-04,  1.1365e-01,  ...,  8.4381e-03,\n",
      "          8.5083e-02, -2.2385e-02],\n",
      "        ...,\n",
      "        [-8.7524e-02, -4.2877e-03, -3.5431e-02,  ..., -6.2622e-02,\n",
      "          2.4199e-04,  8.7769e-02],\n",
      "        [ 8.8318e-02, -6.7673e-03,  4.3030e-02,  ..., -6.0028e-02,\n",
      "         -1.6150e-01, -1.3879e-01],\n",
      "        [-3.7262e-02, -3.1357e-03,  1.8506e-01,  ...,  3.3936e-01,\n",
      "          7.9102e-02, -1.7090e-02]])), ('model.decoder.layers.5.self_attn.q_proj.bias', tensor([ 0.0016, -0.0098,  0.0057,  ..., -0.0281,  0.0271, -0.0198])), ('model.decoder.layers.5.self_attn.out_proj.weight', tensor([[ 0.0214,  0.0254,  0.1136,  ...,  0.0397, -0.0185, -0.0853],\n",
      "        [-0.1061, -0.0003, -0.0640,  ..., -0.0192,  0.0478,  0.0815],\n",
      "        [-0.0189, -0.0401,  0.0135,  ..., -0.0234,  0.0562, -0.0337],\n",
      "        ...,\n",
      "        [ 0.1277, -0.0098,  0.0667,  ..., -0.0396,  0.0617, -0.0629],\n",
      "        [ 0.0906,  0.1194,  0.1010,  ..., -0.0413, -0.0360, -0.0671],\n",
      "        [ 0.0803,  0.0144, -0.0864,  ..., -0.0100, -0.0619, -0.0655]])), ('model.decoder.layers.5.self_attn.out_proj.bias', tensor([ 0.0141,  0.0294,  0.0395,  ..., -0.0341,  0.0544,  0.0502])), ('model.decoder.layers.5.self_attn_layer_norm.weight', tensor([0.9883, 0.9731, 0.9985,  ..., 0.9878, 0.9824, 0.9844])), ('model.decoder.layers.5.self_attn_layer_norm.bias', tensor([ 0.0058, -0.0071,  0.0187,  ...,  0.0040,  0.0304,  0.0227])), ('model.decoder.layers.5.encoder_attn.k_proj.weight', tensor([[-0.0203,  0.1792,  0.1240,  ...,  0.1381,  0.1342,  0.1665],\n",
      "        [-0.0544,  0.1567, -0.0490,  ..., -0.0925,  0.0862, -0.0576],\n",
      "        [ 0.0874, -0.1246, -0.0313,  ..., -0.1120,  0.0170, -0.0299],\n",
      "        ...,\n",
      "        [ 0.1848,  0.1641,  0.0690,  ..., -0.1213,  0.0560,  0.0210],\n",
      "        [ 0.0217,  0.0622,  0.1295,  ..., -0.0441, -0.0369,  0.1698],\n",
      "        [-0.0479,  0.0024, -0.0007,  ...,  0.0626,  0.0056, -0.0629]])), ('model.decoder.layers.5.encoder_attn.k_proj.bias', tensor([ 0.0674, -0.0892, -0.0739,  ..., -0.0221, -0.0974,  0.0186])), ('model.decoder.layers.5.encoder_attn.v_proj.weight', tensor([[ 0.0178,  0.1438,  0.0751,  ..., -0.0160,  0.1078, -0.0415],\n",
      "        [ 0.0424, -0.0483,  0.0350,  ..., -0.1093,  0.0085, -0.0563],\n",
      "        [ 0.0378,  0.0177,  0.0932,  ..., -0.0511, -0.0517,  0.0157],\n",
      "        ...,\n",
      "        [ 0.0464, -0.0927, -0.1247,  ..., -0.0268,  0.1295, -0.0327],\n",
      "        [ 0.1096, -0.0199, -0.0036,  ..., -0.0879,  0.0782, -0.1183],\n",
      "        [-0.0608,  0.1058, -0.0287,  ..., -0.0964, -0.0341,  0.0220]])), ('model.decoder.layers.5.encoder_attn.v_proj.bias', tensor([ 0.0136,  0.0271,  0.0075,  ..., -0.0897, -0.1017,  0.0320])), ('model.decoder.layers.5.encoder_attn.q_proj.weight', tensor([[ 0.0927,  0.1381,  0.0448,  ...,  0.0315, -0.0575,  0.0222],\n",
      "        [ 0.0105,  0.0719,  0.0027,  ...,  0.0224, -0.0027,  0.0895],\n",
      "        [-0.0372,  0.0494, -0.0161,  ..., -0.1624,  0.0113,  0.0636],\n",
      "        ...,\n",
      "        [-0.0786,  0.2576,  0.1414,  ..., -0.0036, -0.1218, -0.0790],\n",
      "        [ 0.0782,  0.0358, -0.0999,  ...,  0.0444, -0.0470,  0.0133],\n",
      "        [ 0.0085,  0.0228,  0.0504,  ...,  0.1154, -0.0148, -0.0544]])), ('model.decoder.layers.5.encoder_attn.q_proj.bias', tensor([-0.0303,  0.0054,  0.0075,  ..., -0.0404, -0.0149, -0.0002])), ('model.decoder.layers.5.encoder_attn.out_proj.weight', tensor([[-0.0016, -0.1377, -0.0164,  ...,  0.0768,  0.0327, -0.0705],\n",
      "        [ 0.1294, -0.0005,  0.0302,  ...,  0.0561, -0.0916,  0.1094],\n",
      "        [-0.0667, -0.0697,  0.0530,  ...,  0.0565, -0.0359, -0.0961],\n",
      "        ...,\n",
      "        [ 0.0139,  0.0572, -0.0684,  ..., -0.0125, -0.0061,  0.0161],\n",
      "        [-0.1212,  0.0519,  0.0029,  ...,  0.0432, -0.0762,  0.0329],\n",
      "        [ 0.0243,  0.0871,  0.0598,  ...,  0.0815,  0.0105, -0.0208]])), ('model.decoder.layers.5.encoder_attn.out_proj.bias', tensor([ 0.0325,  0.0363,  0.0027,  ..., -0.0518,  0.0267, -0.0255])), ('model.decoder.layers.5.encoder_attn_layer_norm.weight', tensor([0.9463, 0.9600, 0.8862,  ..., 0.9385, 0.8701, 0.8979])), ('model.decoder.layers.5.encoder_attn_layer_norm.bias', tensor([ 0.0119,  0.0259,  0.0084,  ..., -0.0025,  0.0162,  0.0202])), ('model.decoder.layers.5.fc1.weight', tensor([[ 0.0304,  0.0254,  0.1165,  ...,  0.0536, -0.0461, -0.0705],\n",
      "        [-0.0100, -0.0196, -0.0609,  ..., -0.0399, -0.0130,  0.0400],\n",
      "        [-0.0778, -0.0816, -0.0057,  ..., -0.1429, -0.0409,  0.0686],\n",
      "        ...,\n",
      "        [-0.1284,  0.0925,  0.0856,  ..., -0.1024,  0.1326,  0.0879],\n",
      "        [-0.0158, -0.1065,  0.0728,  ...,  0.0777, -0.0256, -0.1710],\n",
      "        [-0.0876,  0.0249,  0.1229,  ...,  0.0064,  0.0406, -0.1593]])), ('model.decoder.layers.5.fc1.bias', tensor([-0.0157, -0.0583, -0.0149,  ...,  0.0134, -0.0152, -0.0496])), ('model.decoder.layers.5.fc2.weight', tensor([[-0.0899,  0.0505, -0.0417,  ...,  0.0672, -0.0235,  0.0147],\n",
      "        [-0.0217, -0.0694,  0.0671,  ...,  0.0071,  0.1055,  0.0143],\n",
      "        [-0.0585,  0.0751,  0.0685,  ...,  0.0009, -0.0638,  0.2163],\n",
      "        ...,\n",
      "        [ 0.0087,  0.0816,  0.0685,  ...,  0.0232, -0.0110, -0.1107],\n",
      "        [-0.0217,  0.1407,  0.0598,  ..., -0.1897, -0.0243,  0.1316],\n",
      "        [ 0.0698, -0.0345, -0.0297,  ..., -0.1072,  0.1509,  0.0731]])), ('model.decoder.layers.5.fc2.bias', tensor([-0.0314, -0.0615,  0.0149,  ...,  0.0009,  0.0274, -0.0117])), ('model.decoder.layers.5.final_layer_norm.weight', tensor([0.9438, 0.9858, 0.9829,  ..., 0.9707, 0.9424, 0.9429])), ('model.decoder.layers.5.final_layer_norm.bias', tensor([0.0131, 0.0044, 0.0067,  ..., 0.0046, 0.0199, 0.0104])), ('model.decoder.layers.6.self_attn.k_proj.weight', tensor([[-0.1006, -0.0175, -0.0426,  ..., -0.0411, -0.0522,  0.0710],\n",
      "        [ 0.0327, -0.1127,  0.0066,  ..., -0.2527, -0.2352, -0.0061],\n",
      "        [ 0.0370,  0.1037,  0.0458,  ..., -0.0020,  0.0878,  0.1516],\n",
      "        ...,\n",
      "        [-0.0630, -0.1475, -0.0984,  ...,  0.1043, -0.0361,  0.0855],\n",
      "        [ 0.0856,  0.0142,  0.0739,  ...,  0.0087,  0.0406, -0.1520],\n",
      "        [ 0.0101, -0.0175, -0.1349,  ..., -0.1379, -0.0427, -0.0209]])), ('model.decoder.layers.6.self_attn.k_proj.bias', tensor([-0.0116,  0.0771, -0.1176,  ...,  0.2214,  0.0490, -0.0514])), ('model.decoder.layers.6.self_attn.v_proj.weight', tensor([[ 0.2430,  0.0206,  0.0570,  ..., -0.2581, -0.1410, -0.0143],\n",
      "        [ 0.0622, -0.0443,  0.0075,  ...,  0.1803,  0.1621,  0.1747],\n",
      "        [-0.0434, -0.0090, -0.1168,  ..., -0.1827,  0.1467, -0.0325],\n",
      "        ...,\n",
      "        [-0.0143,  0.1610,  0.0551,  ...,  0.1399, -0.0367,  0.0490],\n",
      "        [-0.0333,  0.0982,  0.0325,  ...,  0.1282, -0.1299, -0.0724],\n",
      "        [ 0.1064,  0.0547,  0.0110,  ...,  0.0399, -0.0121, -0.0267]])), ('model.decoder.layers.6.self_attn.v_proj.bias', tensor([ 0.0206, -0.0127, -0.0116,  ..., -0.0587,  0.0491,  0.0292])), ('model.decoder.layers.6.self_attn.q_proj.weight', tensor([[ 0.0588, -0.0698,  0.0453,  ..., -0.0061, -0.0625, -0.0870],\n",
      "        [ 0.0898,  0.2235, -0.0396,  ..., -0.0193,  0.0529,  0.0256],\n",
      "        [-0.0563, -0.1580,  0.0454,  ..., -0.0110, -0.0829,  0.1927],\n",
      "        ...,\n",
      "        [-0.0483,  0.1011, -0.1154,  ..., -0.1375, -0.0436,  0.0096],\n",
      "        [ 0.1293,  0.0373, -0.0025,  ..., -0.1167, -0.0709, -0.0321],\n",
      "        [ 0.2053, -0.1901, -0.1687,  ..., -0.1447,  0.2367, -0.1835]])), ('model.decoder.layers.6.self_attn.q_proj.bias', tensor([ 0.0127,  0.0127, -0.0413,  ..., -0.0738,  0.0377,  0.0256])), ('model.decoder.layers.6.self_attn.out_proj.weight', tensor([[-0.2037,  0.0465, -0.0792,  ...,  0.0032,  0.0506, -0.0782],\n",
      "        [-0.0887,  0.0825,  0.0042,  ...,  0.1377,  0.1302,  0.0771],\n",
      "        [-0.1292,  0.0543,  0.2219,  ..., -0.1300,  0.0370, -0.1146],\n",
      "        ...,\n",
      "        [ 0.2869, -0.0609,  0.1501,  ...,  0.0254, -0.0008,  0.0784],\n",
      "        [ 0.0944, -0.0461, -0.1592,  ...,  0.0658, -0.0662,  0.0477],\n",
      "        [ 0.0676, -0.0627,  0.1224,  ...,  0.1105,  0.0446, -0.2047]])), ('model.decoder.layers.6.self_attn.out_proj.bias', tensor([ 0.0117,  0.0199, -0.0033,  ..., -0.0305,  0.0174, -0.0060])), ('model.decoder.layers.6.self_attn_layer_norm.weight', tensor([0.9937, 0.9805, 0.9976,  ..., 0.9976, 0.9702, 0.9971])), ('model.decoder.layers.6.self_attn_layer_norm.bias', tensor([ 0.0019, -0.0094,  0.0184,  ...,  0.0100,  0.0323,  0.0257])), ('model.decoder.layers.6.encoder_attn.k_proj.weight', tensor([[ 6.0120e-02,  8.1848e-02, -3.9276e-02,  ..., -4.1138e-02,\n",
      "          1.6998e-02, -1.7029e-02],\n",
      "        [-9.9258e-03, -9.7595e-02,  3.9551e-02,  ...,  5.9509e-02,\n",
      "         -3.5370e-02, -7.9102e-02],\n",
      "        [-6.7688e-02, -1.4587e-01,  1.9714e-02,  ...,  1.3023e-02,\n",
      "          1.0333e-01,  1.1212e-01],\n",
      "        ...,\n",
      "        [-8.2825e-02, -7.6538e-02,  2.1350e-01,  ...,  1.0896e-04,\n",
      "         -1.2183e-01, -2.2812e-02],\n",
      "        [ 8.5388e-02, -9.4238e-02,  1.3220e-01,  ..., -1.0052e-01,\n",
      "         -1.2878e-01, -6.7787e-03],\n",
      "        [ 5.8380e-02,  7.7271e-02, -1.1212e-01,  ..., -3.5736e-02,\n",
      "          1.3391e-01, -4.4495e-02]])), ('model.decoder.layers.6.encoder_attn.k_proj.bias', tensor([-0.0542, -0.0584, -0.4253,  ..., -0.0712, -0.3220,  0.3760])), ('model.decoder.layers.6.encoder_attn.v_proj.weight', tensor([[ 0.0333, -0.1254, -0.0649,  ...,  0.0690, -0.0516, -0.0030],\n",
      "        [ 0.0272, -0.0642,  0.0909,  ...,  0.0640, -0.0373, -0.0156],\n",
      "        [ 0.0057,  0.0212, -0.1110,  ..., -0.1175, -0.1217, -0.0239],\n",
      "        ...,\n",
      "        [ 0.1387, -0.0316,  0.0549,  ...,  0.0523,  0.0551, -0.0482],\n",
      "        [ 0.0065,  0.0641, -0.0860,  ...,  0.0273, -0.1945, -0.0351],\n",
      "        [-0.0824,  0.0202, -0.1597,  ...,  0.0775,  0.0409,  0.0611]])), ('model.decoder.layers.6.encoder_attn.v_proj.bias', tensor([-0.0025, -0.0152, -0.0059,  ..., -0.0171,  0.0046, -0.0186])), ('model.decoder.layers.6.encoder_attn.q_proj.weight', tensor([[-0.0899, -0.0182, -0.0243,  ...,  0.0447,  0.0352, -0.0028],\n",
      "        [-0.1212,  0.0009,  0.1062,  ...,  0.1465,  0.0534, -0.0947],\n",
      "        [-0.0572,  0.1113, -0.0273,  ...,  0.2439, -0.0498,  0.0270],\n",
      "        ...,\n",
      "        [-0.0233, -0.1003,  0.0748,  ..., -0.0828, -0.0986,  0.1381],\n",
      "        [ 0.0509, -0.0127, -0.1184,  ..., -0.0611, -0.1447,  0.0574],\n",
      "        [ 0.1076,  0.0689,  0.0131,  ..., -0.1536, -0.1065, -0.0495]])), ('model.decoder.layers.6.encoder_attn.q_proj.bias', tensor([ 0.0675,  0.0373,  0.0047,  ..., -0.0284, -0.0167,  0.0274])), ('model.decoder.layers.6.encoder_attn.out_proj.weight', tensor([[-0.0041, -0.0149,  0.0480,  ..., -0.0264,  0.0430, -0.0864],\n",
      "        [-0.0609, -0.0088,  0.0518,  ...,  0.0536,  0.1353, -0.0366],\n",
      "        [ 0.0844, -0.0677,  0.0065,  ..., -0.0145,  0.0979, -0.0311],\n",
      "        ...,\n",
      "        [-0.0817, -0.0733,  0.0513,  ..., -0.0358,  0.0352, -0.0607],\n",
      "        [-0.0513, -0.0501,  0.1256,  ..., -0.0703, -0.0351, -0.0182],\n",
      "        [ 0.0158,  0.0782,  0.0068,  ..., -0.0128, -0.1169, -0.0263]])), ('model.decoder.layers.6.encoder_attn.out_proj.bias', tensor([ 0.0108,  0.0050, -0.0071,  ..., -0.0385,  0.0117, -0.0334])), ('model.decoder.layers.6.encoder_attn_layer_norm.weight', tensor([0.9326, 0.9087, 0.9351,  ..., 0.9268, 0.9146, 0.8413])), ('model.decoder.layers.6.encoder_attn_layer_norm.bias', tensor([ 0.0163,  0.0169,  0.0041,  ..., -0.0008,  0.0129,  0.0140])), ('model.decoder.layers.6.fc1.weight', tensor([[ 0.0086, -0.0450, -0.0057,  ...,  0.1633, -0.0176, -0.0970],\n",
      "        [ 0.1124, -0.0157,  0.0529,  ...,  0.1877,  0.1133, -0.1760],\n",
      "        [ 0.2133, -0.0402,  0.0149,  ..., -0.0779, -0.0838, -0.0390],\n",
      "        ...,\n",
      "        [-0.1233, -0.0783,  0.0853,  ..., -0.0019,  0.0525,  0.1215],\n",
      "        [-0.0417, -0.1127, -0.1155,  ..., -0.0012, -0.0704,  0.1578],\n",
      "        [ 0.1008, -0.1183,  0.1135,  ..., -0.0627, -0.0686, -0.0222]])), ('model.decoder.layers.6.fc1.bias', tensor([-0.0345, -0.0580, -0.0374,  ..., -0.0411,  0.0264, -0.0137])), ('model.decoder.layers.6.fc2.weight', tensor([[ 0.0067,  0.0090, -0.0259,  ..., -0.1676, -0.0249, -0.0041],\n",
      "        [-0.0096, -0.0450,  0.1360,  ...,  0.0024,  0.0789,  0.0131],\n",
      "        [ 0.0198,  0.0687,  0.0583,  ...,  0.0527,  0.0626, -0.1285],\n",
      "        ...,\n",
      "        [-0.0728,  0.1476, -0.0913,  ..., -0.0148, -0.1178,  0.0166],\n",
      "        [-0.1429,  0.1278, -0.0424,  ...,  0.0685,  0.0753,  0.0684],\n",
      "        [-0.0014, -0.0972, -0.0517,  ..., -0.0642, -0.1002, -0.1321]])), ('model.decoder.layers.6.fc2.bias', tensor([-0.0134, -0.0361,  0.0270,  ...,  0.0034,  0.0010,  0.0076])), ('model.decoder.layers.6.final_layer_norm.weight', tensor([0.9155, 0.9790, 0.9502,  ..., 0.9497, 0.9546, 0.9185])), ('model.decoder.layers.6.final_layer_norm.bias', tensor([ 0.0179, -0.0008,  0.0069,  ...,  0.0120,  0.0170,  0.0122])), ('model.decoder.layers.7.self_attn.k_proj.weight', tensor([[-0.0408,  0.0538,  0.0062,  ...,  0.1013,  0.1231, -0.0207],\n",
      "        [ 0.1812, -0.2050, -0.0729,  ...,  0.0313, -0.2847, -0.0878],\n",
      "        [-0.1584,  0.0668, -0.0768,  ...,  0.0315, -0.0030,  0.0015],\n",
      "        ...,\n",
      "        [-0.1528,  0.1204, -0.0450,  ..., -0.1689,  0.0972,  0.0550],\n",
      "        [ 0.0352,  0.0894, -0.0646,  ...,  0.0086,  0.0327, -0.0061],\n",
      "        [-0.0177, -0.0441,  0.0392,  ..., -0.1082, -0.0786,  0.1251]])), ('model.decoder.layers.7.self_attn.k_proj.bias', tensor([-0.0651,  0.1516, -0.1067,  ..., -0.0354,  0.2788,  0.0812])), ('model.decoder.layers.7.self_attn.v_proj.weight', tensor([[ 0.0972,  0.0496, -0.0420,  ...,  0.0084, -0.1234,  0.0960],\n",
      "        [ 0.0383, -0.0993,  0.0054,  ..., -0.1085, -0.0803,  0.0116],\n",
      "        [ 0.0773, -0.0490,  0.0258,  ..., -0.0468,  0.1580,  0.0798],\n",
      "        ...,\n",
      "        [ 0.0504,  0.1078,  0.0626,  ..., -0.1162,  0.0163, -0.0159],\n",
      "        [ 0.0316,  0.0281, -0.0253,  ...,  0.0422,  0.0316,  0.1033],\n",
      "        [ 0.0565, -0.0194,  0.1528,  ...,  0.0280, -0.1010,  0.0192]])), ('model.decoder.layers.7.self_attn.v_proj.bias', tensor([-0.0385, -0.0202,  0.0366,  ...,  0.0338, -0.0099, -0.0551])), ('model.decoder.layers.7.self_attn.q_proj.weight', tensor([[-0.2104,  0.0470,  0.1489,  ..., -0.0146,  0.1364, -0.0151],\n",
      "        [ 0.0560, -0.0734,  0.0121,  ...,  0.0793, -0.0443,  0.0393],\n",
      "        [ 0.0724,  0.2278, -0.0272,  ...,  0.0664, -0.0909, -0.0858],\n",
      "        ...,\n",
      "        [-0.0670, -0.1719,  0.1092,  ..., -0.0497, -0.1453,  0.1958],\n",
      "        [ 0.1627, -0.0360,  0.0893,  ..., -0.0179, -0.0891,  0.0040],\n",
      "        [-0.0339, -0.0050, -0.1069,  ...,  0.0847, -0.1262, -0.0361]])), ('model.decoder.layers.7.self_attn.q_proj.bias', tensor([-0.0075,  0.0602, -0.0132,  ..., -0.0191, -0.0013,  0.0036])), ('model.decoder.layers.7.self_attn.out_proj.weight', tensor([[ 0.1083, -0.0798,  0.0270,  ..., -0.0736,  0.0986, -0.0039],\n",
      "        [ 0.0328, -0.1140,  0.0529,  ..., -0.0460, -0.0547,  0.0148],\n",
      "        [ 0.0511,  0.0462, -0.0666,  ...,  0.1174, -0.1779, -0.0739],\n",
      "        ...,\n",
      "        [-0.0246, -0.0991,  0.0233,  ..., -0.0092,  0.0594,  0.0671],\n",
      "        [-0.1110, -0.0562,  0.1227,  ...,  0.0022,  0.1864, -0.1302],\n",
      "        [ 0.0710, -0.0382,  0.0982,  ..., -0.0927,  0.0996,  0.0108]])), ('model.decoder.layers.7.self_attn.out_proj.bias', tensor([ 0.0205,  0.0092, -0.0143,  ..., -0.0098, -0.0027, -0.0059])), ('model.decoder.layers.7.self_attn_layer_norm.weight', tensor([0.9741, 0.9785, 0.9673,  ..., 0.9487, 0.9814, 0.9595])), ('model.decoder.layers.7.self_attn_layer_norm.bias', tensor([ 0.0030, -0.0130,  0.0180,  ...,  0.0160,  0.0309,  0.0295])), ('model.decoder.layers.7.encoder_attn.k_proj.weight', tensor([[-0.0340,  0.0280, -0.1301,  ...,  0.0490, -0.1054,  0.1060],\n",
      "        [-0.1118, -0.1093,  0.0172,  ...,  0.0806, -0.0009,  0.1993],\n",
      "        [-0.0165, -0.0154, -0.0706,  ..., -0.0106, -0.0643, -0.0041],\n",
      "        ...,\n",
      "        [ 0.0157,  0.0595,  0.0651,  ..., -0.0656, -0.0157,  0.1367],\n",
      "        [ 0.0918, -0.1779, -0.0660,  ..., -0.0969, -0.0780,  0.1416],\n",
      "        [-0.0316, -0.0246, -0.1581,  ...,  0.0068,  0.0517, -0.0828]])), ('model.decoder.layers.7.encoder_attn.k_proj.bias', tensor([ 0.1693, -0.0849, -0.1998,  ...,  0.0945, -0.1108, -0.0812])), ('model.decoder.layers.7.encoder_attn.v_proj.weight', tensor([[ 0.0137,  0.0767, -0.0290,  ...,  0.0214, -0.0449, -0.0356],\n",
      "        [ 0.0342,  0.0430, -0.0793,  ...,  0.0172, -0.1769, -0.1066],\n",
      "        [ 0.0163,  0.1206,  0.0931,  ...,  0.0049,  0.0638,  0.0430],\n",
      "        ...,\n",
      "        [ 0.0735,  0.0248, -0.0126,  ..., -0.1171, -0.0165, -0.0900],\n",
      "        [-0.0595,  0.1290,  0.0539,  ...,  0.1770,  0.0565,  0.0038],\n",
      "        [ 0.0489,  0.1099, -0.0852,  ..., -0.0693,  0.1109,  0.1231]])), ('model.decoder.layers.7.encoder_attn.v_proj.bias', tensor([-0.0152,  0.0201,  0.0068,  ..., -0.0308, -0.0030,  0.0090])), ('model.decoder.layers.7.encoder_attn.q_proj.weight', tensor([[-1.8606e-03,  4.9286e-02,  3.9368e-02,  ..., -8.4229e-02,\n",
      "          1.2695e-01,  8.0383e-02],\n",
      "        [-3.5309e-02, -1.5918e-01,  1.4734e-01,  ..., -9.0637e-02,\n",
      "         -1.2573e-01,  7.2510e-02],\n",
      "        [ 6.8604e-02, -1.3245e-01, -2.7161e-02,  ..., -3.5095e-02,\n",
      "          2.6169e-02,  6.9153e-02],\n",
      "        ...,\n",
      "        [ 5.5267e-02,  2.2522e-01, -1.7493e-01,  ..., -1.9348e-02,\n",
      "         -5.1819e-02,  2.1072e-02],\n",
      "        [-1.1963e-02, -1.5247e-01, -1.0205e-01,  ..., -6.9275e-02,\n",
      "         -1.5967e-01,  1.5732e-02],\n",
      "        [ 8.0505e-02,  6.2500e-02,  4.4891e-02,  ..., -1.4699e-04,\n",
      "          3.0212e-02, -3.4271e-02]])), ('model.decoder.layers.7.encoder_attn.q_proj.bias', tensor([ 0.0402,  0.0240, -0.0119,  ...,  0.0150, -0.0086,  0.0353])), ('model.decoder.layers.7.encoder_attn.out_proj.weight', tensor([[ 0.0060, -0.1222,  0.1267,  ...,  0.0710, -0.1552, -0.0145],\n",
      "        [ 0.0166, -0.0742,  0.0391,  ..., -0.0717, -0.0238,  0.0855],\n",
      "        [-0.0161, -0.1048,  0.0260,  ..., -0.0136,  0.0157,  0.0249],\n",
      "        ...,\n",
      "        [ 0.0429,  0.0170,  0.0656,  ..., -0.0146, -0.1508, -0.0831],\n",
      "        [ 0.0675,  0.0275, -0.0890,  ...,  0.0586, -0.1242, -0.1144],\n",
      "        [ 0.0887, -0.0782, -0.0222,  ..., -0.0682,  0.0154, -0.0090]])), ('model.decoder.layers.7.encoder_attn.out_proj.bias', tensor([-0.0073,  0.0215, -0.0461,  ..., -0.0296,  0.0447, -0.0090])), ('model.decoder.layers.7.encoder_attn_layer_norm.weight', tensor([0.9326, 0.9468, 0.9473,  ..., 0.8989, 0.8955, 0.8696])), ('model.decoder.layers.7.encoder_attn_layer_norm.bias', tensor([0.0144, 0.0141, 0.0172,  ..., 0.0011, 0.0133, 0.0141])), ('model.decoder.layers.7.fc1.weight', tensor([[ 0.0210,  0.0467, -0.0742,  ..., -0.0749, -0.0465,  0.0612],\n",
      "        [ 0.0326, -0.1638, -0.2058,  ...,  0.1062,  0.0740,  0.0133],\n",
      "        [-0.0008, -0.0105, -0.1064,  ..., -0.0608,  0.0317, -0.0275],\n",
      "        ...,\n",
      "        [ 0.0316,  0.0682, -0.0680,  ..., -0.0293, -0.0937, -0.0353],\n",
      "        [-0.0591, -0.0629,  0.0531,  ...,  0.1871, -0.0859, -0.0331],\n",
      "        [-0.0871, -0.2113,  0.0353,  ..., -0.1221, -0.0394, -0.0196]])), ('model.decoder.layers.7.fc1.bias', tensor([-0.0072, -0.0291, -0.0528,  ..., -0.0203, -0.0110, -0.0272])), ('model.decoder.layers.7.fc2.weight', tensor([[-0.0196, -0.0565, -0.0021,  ...,  0.0238, -0.0746,  0.0945],\n",
      "        [ 0.0951, -0.0442, -0.0665,  ..., -0.1143,  0.1174, -0.0593],\n",
      "        [ 0.0677, -0.0682, -0.0269,  ...,  0.0716, -0.0563, -0.0525],\n",
      "        ...,\n",
      "        [ 0.0851,  0.1199,  0.0848,  ...,  0.0769, -0.1628,  0.0426],\n",
      "        [-0.0981, -0.0572, -0.0081,  ..., -0.0141,  0.0470,  0.0515],\n",
      "        [-0.0210,  0.0166, -0.0249,  ..., -0.0452,  0.1779, -0.0066]])), ('model.decoder.layers.7.fc2.bias', tensor([ 0.0061, -0.0334, -0.0003,  ..., -0.0267, -0.0356,  0.0276])), ('model.decoder.layers.7.final_layer_norm.weight', tensor([0.9121, 0.9355, 0.9419,  ..., 0.9531, 0.8979, 0.9097])), ('model.decoder.layers.7.final_layer_norm.bias', tensor([ 0.0162, -0.0116,  0.0079,  ...,  0.0219,  0.0138,  0.0115])), ('model.decoder.layers.8.self_attn.k_proj.weight', tensor([[ 0.0211,  0.0301,  0.2144,  ..., -0.0373,  0.1106, -0.0435],\n",
      "        [-0.2089,  0.0737,  0.0023,  ...,  0.0867, -0.0660,  0.2415],\n",
      "        [-0.0391, -0.0840,  0.0164,  ...,  0.0275,  0.1100, -0.0414],\n",
      "        ...,\n",
      "        [ 0.0041,  0.0698, -0.0453,  ..., -0.0687, -0.1716,  0.0745],\n",
      "        [-0.1512,  0.0119, -0.0814,  ..., -0.2157, -0.1760,  0.1801],\n",
      "        [-0.1263,  0.0139, -0.2078,  ..., -0.0993,  0.0215,  0.0836]])), ('model.decoder.layers.8.self_attn.k_proj.bias', tensor([ 0.0465,  0.1434,  0.0399,  ..., -0.0919,  0.0110, -0.0187])), ('model.decoder.layers.8.self_attn.v_proj.weight', tensor([[-5.1300e-02, -1.7480e-01, -1.2402e-01,  ..., -1.2561e-01,\n",
      "         -8.7619e-06,  7.8491e-02],\n",
      "        [ 7.4219e-02, -1.1029e-01, -1.2018e-01,  ..., -2.5854e-01,\n",
      "         -1.8997e-02, -9.3384e-02],\n",
      "        [ 1.1774e-01,  2.5391e-02, -1.4050e-01,  ...,  2.9850e-03,\n",
      "         -6.0349e-03,  1.9958e-02],\n",
      "        ...,\n",
      "        [-1.5976e-02, -5.1514e-02, -1.4206e-02,  ...,  5.7297e-03,\n",
      "          7.4890e-02, -4.6463e-03],\n",
      "        [ 3.4424e-02,  1.5587e-02,  1.7532e-02,  ...,  8.4045e-02,\n",
      "          1.9629e-01,  2.8107e-02],\n",
      "        [-1.0750e-02, -4.5090e-03, -2.0386e-02,  ...,  5.0583e-03,\n",
      "          2.1774e-02,  5.8960e-02]])), ('model.decoder.layers.8.self_attn.v_proj.bias', tensor([-0.0020,  0.0317, -0.0034,  ...,  0.0055,  0.0105, -0.0018])), ('model.decoder.layers.8.self_attn.q_proj.weight', tensor([[ 0.0085, -0.1104,  0.0582,  ..., -0.0670,  0.2074, -0.0854],\n",
      "        [-0.0649,  0.1040, -0.0350,  ...,  0.0933,  0.1648, -0.0146],\n",
      "        [ 0.1156,  0.0980,  0.0731,  ...,  0.0847,  0.0556, -0.0497],\n",
      "        ...,\n",
      "        [ 0.0120,  0.0109,  0.0977,  ..., -0.1288,  0.0635,  0.0398],\n",
      "        [-0.0992,  0.0667,  0.1969,  ...,  0.0638,  0.1565,  0.1647],\n",
      "        [ 0.0334, -0.1049, -0.0158,  ...,  0.0168,  0.0640,  0.0219]])), ('model.decoder.layers.8.self_attn.q_proj.bias', tensor([ 0.0067, -0.0185, -0.0105,  ...,  0.0443,  0.0213, -0.0135])), ('model.decoder.layers.8.self_attn.out_proj.weight', tensor([[ 0.0668, -0.0027, -0.0847,  ...,  0.0051, -0.0617,  0.0779],\n",
      "        [ 0.1772, -0.0042, -0.0122,  ...,  0.0159,  0.1098, -0.0598],\n",
      "        [ 0.0525,  0.0037, -0.0051,  ..., -0.0008, -0.0250,  0.0386],\n",
      "        ...,\n",
      "        [ 0.0914,  0.1501, -0.0443,  ..., -0.0128,  0.1088, -0.0229],\n",
      "        [-0.0585, -0.0470,  0.0139,  ...,  0.0353,  0.1587, -0.1252],\n",
      "        [-0.0533,  0.0983, -0.1130,  ...,  0.0489, -0.0454, -0.0074]])), ('model.decoder.layers.8.self_attn.out_proj.bias', tensor([-0.0047,  0.0250,  0.0032,  ..., -0.0055,  0.0157, -0.0039])), ('model.decoder.layers.8.self_attn_layer_norm.weight', tensor([0.9668, 0.9854, 0.9678,  ..., 0.9502, 0.9390, 0.9556])), ('model.decoder.layers.8.self_attn_layer_norm.bias', tensor([ 0.0015, -0.0174,  0.0213,  ...,  0.0120,  0.0294,  0.0340])), ('model.decoder.layers.8.encoder_attn.k_proj.weight', tensor([[-0.0300,  0.0044,  0.0550,  ..., -0.1196,  0.0357,  0.0757],\n",
      "        [ 0.0729,  0.0597, -0.0963,  ...,  0.0219, -0.1643,  0.0238],\n",
      "        [ 0.0334,  0.0676,  0.0374,  ...,  0.0020,  0.1037, -0.0851],\n",
      "        ...,\n",
      "        [ 0.1973, -0.0487,  0.0546,  ..., -0.0423, -0.1449,  0.0247],\n",
      "        [ 0.0345, -0.1310,  0.1372,  ...,  0.0400, -0.1075,  0.0560],\n",
      "        [ 0.1171,  0.0421,  0.0380,  ...,  0.0806, -0.0536, -0.1023]])), ('model.decoder.layers.8.encoder_attn.k_proj.bias', tensor([ 0.0231,  0.3718,  0.1532,  ..., -0.0245, -0.0272,  0.0741])), ('model.decoder.layers.8.encoder_attn.v_proj.weight', tensor([[-0.0331,  0.0267, -0.0730,  ..., -0.0919,  0.0560,  0.0207],\n",
      "        [-0.0701, -0.0152, -0.0334,  ..., -0.0275,  0.0362, -0.0682],\n",
      "        [-0.0396,  0.0151,  0.0007,  ...,  0.0064,  0.1302,  0.0220],\n",
      "        ...,\n",
      "        [ 0.0201, -0.0364,  0.0223,  ..., -0.1472,  0.0151,  0.0737],\n",
      "        [-0.0628,  0.0084, -0.0436,  ...,  0.1019, -0.1455,  0.0345],\n",
      "        [ 0.0753,  0.0783, -0.0306,  ..., -0.0178,  0.0931,  0.0549]])), ('model.decoder.layers.8.encoder_attn.v_proj.bias', tensor([-0.0553, -0.0291,  0.0157,  ...,  0.0320, -0.0298,  0.0063])), ('model.decoder.layers.8.encoder_attn.q_proj.weight', tensor([[ 0.0412,  0.0119, -0.0845,  ..., -0.0723, -0.0797,  0.0081],\n",
      "        [-0.0053, -0.0572, -0.0314,  ...,  0.0858, -0.0225, -0.0626],\n",
      "        [-0.0705,  0.0396, -0.0020,  ...,  0.0248,  0.0911,  0.0340],\n",
      "        ...,\n",
      "        [ 0.0433, -0.1350, -0.0918,  ..., -0.0667,  0.0118, -0.0067],\n",
      "        [-0.0267, -0.0586,  0.0549,  ...,  0.0545, -0.0721, -0.0353],\n",
      "        [-0.1493,  0.0916, -0.0925,  ..., -0.0071, -0.0276, -0.0218]])), ('model.decoder.layers.8.encoder_attn.q_proj.bias', tensor([-0.0040, -0.0253,  0.0118,  ..., -0.0609,  0.0297, -0.0213])), ('model.decoder.layers.8.encoder_attn.out_proj.weight', tensor([[ 0.1157, -0.1235,  0.0156,  ...,  0.1016,  0.0059, -0.0158],\n",
      "        [-0.0982,  0.0267,  0.0195,  ..., -0.0615,  0.0219, -0.0331],\n",
      "        [-0.0137,  0.0527, -0.0011,  ...,  0.0754, -0.0227, -0.1196],\n",
      "        ...,\n",
      "        [-0.0485, -0.0871, -0.1069,  ..., -0.0519,  0.0222, -0.0754],\n",
      "        [-0.0495,  0.0556,  0.0173,  ..., -0.0612, -0.0303, -0.0420],\n",
      "        [ 0.1290, -0.0179,  0.1169,  ..., -0.1364,  0.1249, -0.1003]])), ('model.decoder.layers.8.encoder_attn.out_proj.bias', tensor([-0.0190,  0.0009, -0.0423,  ...,  0.0008,  0.0349,  0.0077])), ('model.decoder.layers.8.encoder_attn_layer_norm.weight', tensor([0.9004, 0.9189, 0.8882,  ..., 0.8818, 0.9155, 0.8657])), ('model.decoder.layers.8.encoder_attn_layer_norm.bias', tensor([0.0144, 0.0040, 0.0112,  ..., 0.0076, 0.0109, 0.0141])), ('model.decoder.layers.8.fc1.weight', tensor([[-0.0089,  0.1204,  0.0907,  ..., -0.0649, -0.0534,  0.0241],\n",
      "        [-0.1102,  0.0021,  0.0619,  ...,  0.0660, -0.0193,  0.0096],\n",
      "        [ 0.0371, -0.1338, -0.0392,  ...,  0.0292, -0.1050, -0.1475],\n",
      "        ...,\n",
      "        [-0.0210,  0.0194, -0.0675,  ..., -0.0262, -0.0459, -0.0203],\n",
      "        [-0.0238,  0.0525, -0.0812,  ..., -0.0059, -0.0192, -0.0277],\n",
      "        [ 0.0377,  0.0595, -0.1379,  ...,  0.0455, -0.0290, -0.0651]])), ('model.decoder.layers.8.fc1.bias', tensor([-3.0151e-02,  1.0025e-02,  1.3232e-05,  ...,  1.1604e-02,\n",
      "        -2.3804e-02, -6.6528e-03])), ('model.decoder.layers.8.fc2.weight', tensor([[-0.0472,  0.0666, -0.1255,  ...,  0.0599,  0.0137, -0.1600],\n",
      "        [-0.0538,  0.0618,  0.0543,  ...,  0.0548, -0.1069,  0.0201],\n",
      "        [-0.1633, -0.0692,  0.0991,  ..., -0.0220, -0.0524,  0.0129],\n",
      "        ...,\n",
      "        [ 0.0487, -0.0356, -0.0402,  ...,  0.1187, -0.0778, -0.0771],\n",
      "        [ 0.0371,  0.0043, -0.0115,  ..., -0.0942, -0.0594,  0.0869],\n",
      "        [-0.0610,  0.0402,  0.0336,  ...,  0.0519,  0.1550, -0.0087]])), ('model.decoder.layers.8.fc2.bias', tensor([ 0.0272, -0.0125, -0.0114,  ..., -0.0128, -0.0895,  0.0251])), ('model.decoder.layers.8.final_layer_norm.weight', tensor([0.8687, 0.9272, 1.0010,  ..., 0.9194, 0.8726, 0.8892])), ('model.decoder.layers.8.final_layer_norm.bias', tensor([ 0.0240, -0.0208,  0.0076,  ...,  0.0293,  0.0077,  0.0132])), ('model.decoder.layers.9.self_attn.k_proj.weight', tensor([[-0.0898, -0.0148,  0.0806,  ...,  0.0393, -0.0263,  0.0175],\n",
      "        [ 0.0544,  0.0850,  0.1437,  ..., -0.0781, -0.0096, -0.0652],\n",
      "        [-0.0274,  0.0215,  0.0371,  ..., -0.0422,  0.1043,  0.0565],\n",
      "        ...,\n",
      "        [-0.0222, -0.1624, -0.0434,  ..., -0.0397,  0.0147, -0.1875],\n",
      "        [-0.0059,  0.0500,  0.2301,  ...,  0.0806,  0.0434, -0.0761],\n",
      "        [-0.0698,  0.1655,  0.0551,  ..., -0.0410, -0.0839,  0.1367]])), ('model.decoder.layers.9.self_attn.k_proj.bias', tensor([ 0.0722,  0.0536, -0.0903,  ..., -0.0522,  0.0884, -0.0023])), ('model.decoder.layers.9.self_attn.v_proj.weight', tensor([[ 0.0395, -0.0258,  0.0493,  ..., -0.1593,  0.0093,  0.0388],\n",
      "        [ 0.1117, -0.1765,  0.0461,  ...,  0.0631,  0.0800,  0.1210],\n",
      "        [-0.0920, -0.0435, -0.0424,  ..., -0.0536,  0.0461,  0.1088],\n",
      "        ...,\n",
      "        [ 0.0648,  0.1266, -0.0247,  ...,  0.0894,  0.0362, -0.1169],\n",
      "        [-0.1470, -0.0554, -0.0926,  ..., -0.0829,  0.1004, -0.0480],\n",
      "        [ 0.0257,  0.0477,  0.0120,  ...,  0.0513,  0.0721,  0.0162]])), ('model.decoder.layers.9.self_attn.v_proj.bias', tensor([ 0.0300, -0.0956,  0.0709,  ...,  0.0157,  0.0203,  0.0110])), ('model.decoder.layers.9.self_attn.q_proj.weight', tensor([[-0.0773,  0.0424,  0.0253,  ...,  0.0388, -0.0679,  0.1115],\n",
      "        [-0.0161,  0.0056,  0.0428,  ..., -0.1632,  0.0822,  0.0104],\n",
      "        [-0.1526, -0.0452,  0.1094,  ..., -0.0749, -0.0421,  0.1793],\n",
      "        ...,\n",
      "        [-0.0803, -0.0627, -0.0381,  ..., -0.0757, -0.0004, -0.2390],\n",
      "        [ 0.0350, -0.0155,  0.0797,  ...,  0.1340, -0.1527, -0.0182],\n",
      "        [-0.0069, -0.0206,  0.1281,  ...,  0.1332, -0.2153,  0.1648]])), ('model.decoder.layers.9.self_attn.q_proj.bias', tensor([ 0.0003,  0.0117,  0.0035,  ...,  0.0024, -0.0293, -0.0110])), ('model.decoder.layers.9.self_attn.out_proj.weight', tensor([[ 0.0831, -0.0741,  0.1198,  ..., -0.1008,  0.0682, -0.0411],\n",
      "        [ 0.1024, -0.0173,  0.1127,  ...,  0.0217, -0.0007,  0.0864],\n",
      "        [ 0.1254, -0.1115, -0.0302,  ..., -0.0215,  0.0422, -0.0358],\n",
      "        ...,\n",
      "        [-0.0242,  0.0690, -0.0367,  ...,  0.0249,  0.0291, -0.0529],\n",
      "        [-0.0494,  0.0213,  0.0349,  ..., -0.0103, -0.0500,  0.0225],\n",
      "        [ 0.0488,  0.0247, -0.1910,  ..., -0.0499, -0.0883, -0.1047]])), ('model.decoder.layers.9.self_attn.out_proj.bias', tensor([-0.0253, -0.0104,  0.0276,  ..., -0.0022,  0.0308,  0.0116])), ('model.decoder.layers.9.self_attn_layer_norm.weight', tensor([0.9165, 0.9468, 0.8994,  ..., 0.9004, 0.8936, 0.9082])), ('model.decoder.layers.9.self_attn_layer_norm.bias', tensor([ 0.0018, -0.0201,  0.0223,  ...,  0.0124,  0.0250,  0.0374])), ('model.decoder.layers.9.encoder_attn.k_proj.weight', tensor([[ 0.0778,  0.0091,  0.0757,  ...,  0.1144,  0.0022,  0.0034],\n",
      "        [-0.0490,  0.0786,  0.0114,  ..., -0.0238, -0.0864,  0.0223],\n",
      "        [ 0.0493,  0.0231,  0.0766,  ...,  0.0146, -0.0541,  0.0236],\n",
      "        ...,\n",
      "        [-0.0153, -0.0599, -0.0286,  ...,  0.0634,  0.0259, -0.0842],\n",
      "        [ 0.0455,  0.0041,  0.0317,  ..., -0.0042, -0.1261,  0.0950],\n",
      "        [-0.0126, -0.0317, -0.0484,  ..., -0.0305,  0.1232,  0.0137]])), ('model.decoder.layers.9.encoder_attn.k_proj.bias', tensor([-0.1072, -0.1465,  0.0913,  ...,  0.5679, -0.2559, -0.3398])), ('model.decoder.layers.9.encoder_attn.v_proj.weight', tensor([[ 0.0204, -0.0723, -0.0063,  ..., -0.0243, -0.0088,  0.0563],\n",
      "        [ 0.0178,  0.0293, -0.0483,  ..., -0.0516,  0.0343,  0.0399],\n",
      "        [-0.1033,  0.0416,  0.0325,  ...,  0.0461,  0.0233, -0.0322],\n",
      "        ...,\n",
      "        [-0.0776,  0.0831,  0.0901,  ..., -0.0407, -0.0370,  0.1371],\n",
      "        [-0.0427, -0.0300,  0.0643,  ...,  0.0241, -0.0753, -0.0170],\n",
      "        [-0.0205, -0.0438,  0.1912,  ..., -0.0192, -0.0318,  0.1243]])), ('model.decoder.layers.9.encoder_attn.v_proj.bias', tensor([ 0.0100, -0.0243, -0.0077,  ...,  0.0538,  0.0424,  0.0419])), ('model.decoder.layers.9.encoder_attn.q_proj.weight', tensor([[ 0.0635, -0.1121, -0.0854,  ...,  0.0185, -0.0848,  0.0186],\n",
      "        [-0.0836,  0.0530, -0.0358,  ...,  0.0954,  0.0468, -0.0598],\n",
      "        [-0.0318, -0.1129, -0.0338,  ...,  0.0419,  0.1180, -0.0254],\n",
      "        ...,\n",
      "        [-0.1016,  0.0459, -0.0033,  ..., -0.0220, -0.0410, -0.0776],\n",
      "        [-0.0195, -0.0132,  0.0990,  ..., -0.0290,  0.1251,  0.0526],\n",
      "        [-0.0594,  0.0274,  0.1061,  ...,  0.0183, -0.0677,  0.0645]])), ('model.decoder.layers.9.encoder_attn.q_proj.bias', tensor([-0.0208, -0.0301, -0.0060,  ...,  0.0129, -0.0103, -0.0209])), ('model.decoder.layers.9.encoder_attn.out_proj.weight', tensor([[ 0.0418,  0.0155,  0.0126,  ...,  0.0369,  0.0366, -0.0016],\n",
      "        [ 0.0722, -0.0216,  0.1265,  ...,  0.0891,  0.0224,  0.0243],\n",
      "        [ 0.0358,  0.0085, -0.0136,  ..., -0.1182, -0.0543,  0.0320],\n",
      "        ...,\n",
      "        [ 0.0375, -0.0439,  0.0415,  ...,  0.0980,  0.0856,  0.0051],\n",
      "        [-0.0289,  0.0045,  0.0689,  ..., -0.0106,  0.0793, -0.0458],\n",
      "        [ 0.0077,  0.0944,  0.0675,  ...,  0.0839, -0.0237, -0.0511]])), ('model.decoder.layers.9.encoder_attn.out_proj.bias', tensor([-0.0167,  0.0003, -0.0061,  ..., -0.0166, -0.0085,  0.0088])), ('model.decoder.layers.9.encoder_attn_layer_norm.weight', tensor([0.9058, 0.9077, 0.8574,  ..., 0.9258, 0.8950, 0.8608])), ('model.decoder.layers.9.encoder_attn_layer_norm.bias', tensor([0.0136, 0.0070, 0.0124,  ..., 0.0071, 0.0057, 0.0107])), ('model.decoder.layers.9.fc1.weight', tensor([[ 0.0900, -0.0360, -0.0224,  ...,  0.0711, -0.1003,  0.0181],\n",
      "        [-0.0047, -0.0885,  0.0396,  ..., -0.0400,  0.0623,  0.0232],\n",
      "        [-0.0542,  0.0059, -0.0318,  ...,  0.0855, -0.1781, -0.0098],\n",
      "        ...,\n",
      "        [ 0.0544, -0.1824,  0.0321,  ...,  0.0352,  0.0361,  0.0479],\n",
      "        [-0.0033, -0.0504,  0.0066,  ...,  0.0564,  0.0119, -0.1066],\n",
      "        [ 0.0908, -0.1061,  0.1382,  ...,  0.0295, -0.0462, -0.1209]])), ('model.decoder.layers.9.fc1.bias', tensor([-0.0356, -0.0004, -0.0265,  ..., -0.0485, -0.0124, -0.0229])), ('model.decoder.layers.9.fc2.weight', tensor([[-0.0370,  0.0742,  0.0755,  ..., -0.1041,  0.0215, -0.0753],\n",
      "        [-0.0220,  0.0113,  0.1287,  ..., -0.0957,  0.0638, -0.1517],\n",
      "        [-0.0242, -0.0836,  0.0137,  ...,  0.1393,  0.1257,  0.0476],\n",
      "        ...,\n",
      "        [ 0.0298,  0.0037, -0.0427,  ...,  0.0427, -0.0450,  0.0970],\n",
      "        [ 0.0469,  0.0575, -0.0583,  ..., -0.1583,  0.0503, -0.0491],\n",
      "        [-0.0025,  0.0185, -0.0277,  ...,  0.0737, -0.1316, -0.1992]])), ('model.decoder.layers.9.fc2.bias', tensor([ 0.0431,  0.0612, -0.0298,  ..., -0.0173, -0.1229,  0.0027])), ('model.decoder.layers.9.final_layer_norm.weight', tensor([0.8140, 0.9209, 1.0049,  ..., 0.8740, 0.7954, 0.8193])), ('model.decoder.layers.9.final_layer_norm.bias', tensor([ 0.0333, -0.0421,  0.0057,  ...,  0.0370,  0.0016,  0.0183])), ('model.decoder.layers.10.self_attn.k_proj.weight', tensor([[-0.0948,  0.1432, -0.0302,  ..., -0.0417, -0.0257, -0.0969],\n",
      "        [ 0.0251,  0.0070, -0.0120,  ..., -0.0740,  0.0812, -0.0692],\n",
      "        [ 0.0348,  0.0748,  0.0118,  ...,  0.0883, -0.1249, -0.0905],\n",
      "        ...,\n",
      "        [-0.0865,  0.0023, -0.0603,  ...,  0.0488,  0.0510, -0.0277],\n",
      "        [-0.1008, -0.1243,  0.1765,  ..., -0.0308, -0.1305, -0.0961],\n",
      "        [-0.0276, -0.0406,  0.1113,  ..., -0.0068, -0.1031,  0.0901]])), ('model.decoder.layers.10.self_attn.k_proj.bias', tensor([-2.0065e-02,  4.7760e-02,  6.3477e-02,  ..., -2.9831e-02,\n",
      "        -2.6001e-02,  1.6093e-05])), ('model.decoder.layers.10.self_attn.v_proj.weight', tensor([[ 0.1595, -0.0587, -0.0226,  ...,  0.1470, -0.1748,  0.1722],\n",
      "        [ 0.1855,  0.0922,  0.0379,  ..., -0.0216, -0.1371, -0.1310],\n",
      "        [-0.0471, -0.0251, -0.1092,  ...,  0.0360,  0.1476, -0.0634],\n",
      "        ...,\n",
      "        [ 0.0093,  0.0779,  0.0800,  ..., -0.0244, -0.1075,  0.1954],\n",
      "        [ 0.0561,  0.0123, -0.0616,  ...,  0.0500,  0.0902,  0.0726],\n",
      "        [ 0.0330, -0.0311, -0.0782,  ..., -0.0053,  0.0389,  0.1239]])), ('model.decoder.layers.10.self_attn.v_proj.bias', tensor([ 0.0044, -0.0257,  0.0101,  ..., -0.0077, -0.0345,  0.0134])), ('model.decoder.layers.10.self_attn.q_proj.weight', tensor([[ 0.0243,  0.1399, -0.1045,  ..., -0.0771, -0.0146, -0.0381],\n",
      "        [-0.0477, -0.0107, -0.1570,  ...,  0.0308,  0.1469,  0.0030],\n",
      "        [-0.0301,  0.1566, -0.0567,  ...,  0.0301,  0.0472,  0.0251],\n",
      "        ...,\n",
      "        [-0.0736, -0.1172,  0.0090,  ...,  0.0498,  0.0729, -0.0226],\n",
      "        [ 0.1199,  0.0779,  0.0481,  ...,  0.0042, -0.0255,  0.0447],\n",
      "        [-0.0210,  0.0012,  0.2272,  ..., -0.0132,  0.0219,  0.1257]])), ('model.decoder.layers.10.self_attn.q_proj.bias', tensor([-0.0049,  0.0107, -0.0266,  ..., -0.0451, -0.0622,  0.0175])), ('model.decoder.layers.10.self_attn.out_proj.weight', tensor([[ 0.0500,  0.0308, -0.1309,  ..., -0.0439, -0.0616,  0.1012],\n",
      "        [ 0.0860,  0.0109, -0.0410,  ...,  0.0809,  0.1707,  0.0546],\n",
      "        [ 0.0306,  0.1202,  0.0781,  ...,  0.0948,  0.0556, -0.0018],\n",
      "        ...,\n",
      "        [-0.0634,  0.0850,  0.0375,  ..., -0.0748,  0.0283, -0.0457],\n",
      "        [-0.0008, -0.1365,  0.0691,  ...,  0.0828, -0.0778, -0.0289],\n",
      "        [-0.0301, -0.0392, -0.0137,  ...,  0.0061, -0.0488,  0.0927]])), ('model.decoder.layers.10.self_attn.out_proj.bias', tensor([-0.0789,  0.0008,  0.0604,  ...,  0.0152,  0.0384,  0.0423])), ('model.decoder.layers.10.self_attn_layer_norm.weight', tensor([0.8604, 0.9067, 0.8423,  ..., 0.8774, 0.8501, 0.9053])), ('model.decoder.layers.10.self_attn_layer_norm.bias', tensor([ 0.0056, -0.0289,  0.0195,  ...,  0.0144,  0.0170,  0.0421])), ('model.decoder.layers.10.encoder_attn.k_proj.weight', tensor([[-3.5553e-02,  3.8940e-02, -6.1226e-04,  ..., -9.1492e-02,\n",
      "         -2.3666e-02, -1.3086e-01],\n",
      "        [ 1.6479e-03,  2.8503e-02,  5.0507e-02,  ...,  3.7933e-02,\n",
      "          3.7964e-02, -2.6993e-02],\n",
      "        [-6.7566e-02,  2.6215e-02,  2.5070e-02,  ..., -4.4342e-02,\n",
      "         -1.6983e-02, -3.0457e-02],\n",
      "        ...,\n",
      "        [ 4.1138e-02,  1.3257e-01,  5.5634e-02,  ...,  4.5929e-02,\n",
      "         -1.5991e-02, -2.5467e-02],\n",
      "        [ 1.9836e-04,  1.0852e-01, -7.3792e-02,  ...,  7.5928e-02,\n",
      "         -3.6652e-02,  4.9171e-03],\n",
      "        [-1.1188e-01,  2.7786e-02,  6.7444e-02,  ...,  8.9765e-05,\n",
      "          1.2600e-04,  8.6304e-02]])), ('model.decoder.layers.10.encoder_attn.k_proj.bias', tensor([-0.1569, -0.0443,  0.0665,  ...,  0.1598, -0.0215, -0.4036])), ('model.decoder.layers.10.encoder_attn.v_proj.weight', tensor([[ 0.0795, -0.0246, -0.1447,  ..., -0.0150,  0.0514, -0.0432],\n",
      "        [-0.0141, -0.0348,  0.0559,  ..., -0.0339,  0.0075,  0.0688],\n",
      "        [ 0.0795,  0.0227,  0.0945,  ...,  0.0138, -0.0384,  0.0043],\n",
      "        ...,\n",
      "        [-0.0430, -0.0671,  0.1547,  ...,  0.0262, -0.1044, -0.0013],\n",
      "        [-0.1226, -0.0323,  0.0168,  ...,  0.0856, -0.0640,  0.0050],\n",
      "        [ 0.0077,  0.0156,  0.0265,  ..., -0.0744, -0.0966, -0.0399]])), ('model.decoder.layers.10.encoder_attn.v_proj.bias', tensor([-0.0681,  0.0356,  0.0549,  ...,  0.0584, -0.0160,  0.0622])), ('model.decoder.layers.10.encoder_attn.q_proj.weight', tensor([[-0.0425,  0.0673, -0.0016,  ...,  0.0186, -0.0449, -0.0127],\n",
      "        [ 0.0142, -0.1091, -0.0630,  ..., -0.1440,  0.0118,  0.0708],\n",
      "        [-0.0396, -0.0317, -0.1034,  ...,  0.0531,  0.0877,  0.0496],\n",
      "        ...,\n",
      "        [-0.0129, -0.1764,  0.0383,  ...,  0.0517, -0.0154, -0.0155],\n",
      "        [ 0.0383,  0.0443,  0.0303,  ...,  0.0406,  0.1074,  0.0822],\n",
      "        [-0.0469, -0.0255, -0.0311,  ..., -0.0364,  0.0443,  0.0595]])), ('model.decoder.layers.10.encoder_attn.q_proj.bias', tensor([ 0.1158,  0.0505, -0.0281,  ...,  0.0469,  0.0139,  0.0283])), ('model.decoder.layers.10.encoder_attn.out_proj.weight', tensor([[ 0.1010,  0.0849,  0.0092,  ...,  0.0395,  0.0691,  0.1020],\n",
      "        [ 0.1267, -0.0413, -0.0744,  ..., -0.0856, -0.1405, -0.0585],\n",
      "        [-0.1051,  0.0252,  0.0330,  ..., -0.0116, -0.0906, -0.0462],\n",
      "        ...,\n",
      "        [ 0.0667, -0.0788,  0.0081,  ..., -0.0965,  0.0557, -0.0849],\n",
      "        [-0.0647, -0.0732,  0.0285,  ...,  0.0517, -0.0809,  0.0626],\n",
      "        [-0.0401, -0.0292, -0.0043,  ..., -0.0340, -0.0345, -0.0533]])), ('model.decoder.layers.10.encoder_attn.out_proj.bias', tensor([-0.0380, -0.0045, -0.0133,  ...,  0.0100,  0.0274,  0.0177])), ('model.decoder.layers.10.encoder_attn_layer_norm.weight', tensor([0.8394, 0.8950, 0.8418,  ..., 0.8770, 0.8730, 0.8276])), ('model.decoder.layers.10.encoder_attn_layer_norm.bias', tensor([ 0.0247,  0.0014,  0.0195,  ..., -0.0150,  0.0053,  0.0205])), ('model.decoder.layers.10.fc1.weight', tensor([[ 0.0411,  0.0145, -0.0320,  ..., -0.1458, -0.0807,  0.0271],\n",
      "        [ 0.0142,  0.0513,  0.0147,  ..., -0.0830,  0.0396, -0.0423],\n",
      "        [ 0.0417, -0.1345, -0.0016,  ..., -0.0400, -0.0534, -0.0129],\n",
      "        ...,\n",
      "        [-0.0402,  0.1169, -0.0011,  ..., -0.0236, -0.0431, -0.1218],\n",
      "        [-0.0561, -0.0643, -0.0417,  ...,  0.0196, -0.0488,  0.0186],\n",
      "        [ 0.1406, -0.0486,  0.0197,  ...,  0.2264, -0.0138,  0.0657]])), ('model.decoder.layers.10.fc1.bias', tensor([-2.2385e-02,  1.6093e-06, -2.1469e-02,  ..., -3.4943e-02,\n",
      "        -2.2449e-03,  1.2779e-03])), ('model.decoder.layers.10.fc2.weight', tensor([[ 0.1383,  0.1014,  0.1030,  ...,  0.0509,  0.1370,  0.0290],\n",
      "        [-0.0245, -0.0211,  0.0776,  ...,  0.1388, -0.0820, -0.0191],\n",
      "        [-0.0482, -0.0808,  0.0319,  ..., -0.0551,  0.0335,  0.0207],\n",
      "        ...,\n",
      "        [ 0.1166,  0.0735,  0.1549,  ..., -0.1711,  0.0628, -0.0153],\n",
      "        [ 0.0178, -0.0491, -0.0446,  ..., -0.0345,  0.0238,  0.0344],\n",
      "        [-0.0594,  0.0609, -0.0865,  ..., -0.0139,  0.0587,  0.0876]])), ('model.decoder.layers.10.fc2.bias', tensor([ 0.0314,  0.0441,  0.0141,  ..., -0.0309, -0.1354,  0.0622])), ('model.decoder.layers.10.final_layer_norm.weight', tensor([0.7812, 0.8364, 1.0029,  ..., 0.8364, 0.7871, 0.7817])), ('model.decoder.layers.10.final_layer_norm.bias', tensor([ 0.0408, -0.0576,  0.0018,  ...,  0.0390, -0.0084,  0.0237])), ('model.decoder.layers.11.self_attn.k_proj.weight', tensor([[-0.0412,  0.0714,  0.0879,  ..., -0.0378, -0.0266,  0.0265],\n",
      "        [-0.0131,  0.0167, -0.0028,  ...,  0.1183,  0.0043, -0.0438],\n",
      "        [ 0.0839,  0.0655, -0.0011,  ..., -0.0205, -0.0322,  0.0662],\n",
      "        ...,\n",
      "        [ 0.1313, -0.0147, -0.0042,  ...,  0.0231,  0.0379, -0.0591],\n",
      "        [ 0.0662,  0.0416,  0.1321,  ..., -0.0052, -0.0760,  0.0497],\n",
      "        [-0.0569,  0.0670,  0.0333,  ...,  0.0515, -0.1373, -0.2435]])), ('model.decoder.layers.11.self_attn.k_proj.bias', tensor([ 0.1013, -0.2612,  0.0457,  ...,  0.2371,  0.2179,  0.2734])), ('model.decoder.layers.11.self_attn.v_proj.weight', tensor([[ 0.2101, -0.1537,  0.0098,  ...,  0.0104,  0.0437, -0.0489],\n",
      "        [ 0.0924, -0.0814,  0.0290,  ..., -0.0135, -0.0545,  0.0070],\n",
      "        [ 0.0030,  0.0891, -0.0335,  ..., -0.0678, -0.0703, -0.1138],\n",
      "        ...,\n",
      "        [-0.0402, -0.0690,  0.0382,  ..., -0.0627,  0.0081,  0.0286],\n",
      "        [-0.1228,  0.0062,  0.0178,  ..., -0.0199,  0.0270, -0.0228],\n",
      "        [ 0.0270, -0.0618, -0.0023,  ...,  0.0010, -0.0615,  0.0791]])), ('model.decoder.layers.11.self_attn.v_proj.bias', tensor([ 0.0148,  0.0234, -0.0210,  ...,  0.0265,  0.0099,  0.0016])), ('model.decoder.layers.11.self_attn.q_proj.weight', tensor([[-0.0317, -0.0899,  0.0551,  ...,  0.0643,  0.0754,  0.0574],\n",
      "        [ 0.0347, -0.0127,  0.0468,  ...,  0.0211,  0.1267, -0.0565],\n",
      "        [-0.1153, -0.0174, -0.0377,  ..., -0.0081,  0.0315,  0.1912],\n",
      "        ...,\n",
      "        [ 0.1017,  0.1042,  0.0248,  ...,  0.0047,  0.0281, -0.0086],\n",
      "        [ 0.0137,  0.0114, -0.0055,  ..., -0.0427, -0.1411,  0.0748],\n",
      "        [ 0.0208, -0.0111, -0.0453,  ...,  0.0551, -0.0150, -0.0563]])), ('model.decoder.layers.11.self_attn.q_proj.bias', tensor([-0.0414,  0.0027, -0.0049,  ..., -0.0277,  0.0141,  0.0049])), ('model.decoder.layers.11.self_attn.out_proj.weight', tensor([[ 0.0374, -0.0097,  0.0732,  ..., -0.0654, -0.0487,  0.0428],\n",
      "        [-0.0180, -0.0432,  0.0275,  ..., -0.0834, -0.0557,  0.0292],\n",
      "        [ 0.0955, -0.0443, -0.0674,  ..., -0.0770,  0.0478, -0.0080],\n",
      "        ...,\n",
      "        [-0.0323,  0.0316,  0.0601,  ..., -0.1384, -0.0455,  0.0652],\n",
      "        [-0.0068, -0.0571,  0.0060,  ..., -0.0995, -0.0160, -0.0044],\n",
      "        [-0.0287, -0.0258, -0.0366,  ...,  0.0027,  0.0242,  0.0367]])), ('model.decoder.layers.11.self_attn.out_proj.bias', tensor([-0.1240,  0.0490,  0.0618,  ..., -0.0135,  0.0443,  0.0025])), ('model.decoder.layers.11.self_attn_layer_norm.weight', tensor([0.7490, 0.8096, 0.7544,  ..., 0.7759, 0.7773, 0.8291])), ('model.decoder.layers.11.self_attn_layer_norm.bias', tensor([ 0.0155, -0.0449,  0.0089,  ...,  0.0166, -0.0093,  0.0371])), ('model.decoder.layers.11.encoder_attn.k_proj.weight', tensor([[-0.0167,  0.0320, -0.1315,  ..., -0.0490, -0.0629, -0.0225],\n",
      "        [-0.0374, -0.1021, -0.0643,  ..., -0.0465, -0.1015, -0.0100],\n",
      "        [-0.0588, -0.0644, -0.1631,  ...,  0.1471,  0.0167,  0.0604],\n",
      "        ...,\n",
      "        [ 0.0157,  0.0552, -0.0237,  ..., -0.0237,  0.0644, -0.0430],\n",
      "        [ 0.0365,  0.0101, -0.0524,  ...,  0.0678, -0.0562, -0.0350],\n",
      "        [-0.0083,  0.0042,  0.0767,  ...,  0.0040,  0.1005,  0.0447]])), ('model.decoder.layers.11.encoder_attn.k_proj.bias', tensor([-0.0182,  0.2664, -0.0377,  ...,  0.0316, -0.0555,  0.0319])), ('model.decoder.layers.11.encoder_attn.v_proj.weight', tensor([[ 0.0985, -0.3955,  0.0368,  ...,  0.0069, -0.0247, -0.1125],\n",
      "        [-0.1255,  0.0641,  0.3064,  ..., -0.0153, -0.1221,  0.0935],\n",
      "        [ 0.0461, -0.1002,  0.2291,  ...,  0.1835, -0.0715, -0.0083],\n",
      "        ...,\n",
      "        [-0.0438,  0.0986, -0.0200,  ...,  0.0040, -0.0380, -0.0623],\n",
      "        [ 0.1250,  0.0789, -0.0299,  ...,  0.0474,  0.0386, -0.0703],\n",
      "        [-0.0434,  0.0316,  0.0991,  ..., -0.0299,  0.0939,  0.0500]])), ('model.decoder.layers.11.encoder_attn.v_proj.bias', tensor([-0.0123,  0.0461,  0.0106,  ...,  0.0591, -0.0131,  0.0649])), ('model.decoder.layers.11.encoder_attn.q_proj.weight', tensor([[-0.0354, -0.0978,  0.1109,  ...,  0.0950,  0.0718, -0.0888],\n",
      "        [-0.0969,  0.1069, -0.0091,  ..., -0.0559, -0.0103, -0.0682],\n",
      "        [ 0.0705, -0.0164,  0.1898,  ..., -0.0745,  0.0353,  0.1764],\n",
      "        ...,\n",
      "        [-0.0221, -0.0118,  0.0483,  ..., -0.0900,  0.0163, -0.0485],\n",
      "        [-0.0296,  0.0064, -0.0146,  ...,  0.0017,  0.0507, -0.2007],\n",
      "        [ 0.0505, -0.0626, -0.0248,  ...,  0.0009, -0.0388, -0.0735]])), ('model.decoder.layers.11.encoder_attn.q_proj.bias', tensor([ 0.0009,  0.0895,  0.0073,  ...,  0.0481, -0.0474, -0.0749])), ('model.decoder.layers.11.encoder_attn.out_proj.weight', tensor([[-0.0520,  0.0704,  0.0106,  ...,  0.0428,  0.1002, -0.0852],\n",
      "        [ 0.1760,  0.0587, -0.0196,  ...,  0.1301,  0.0273,  0.0337],\n",
      "        [ 0.0022,  0.0060, -0.1359,  ...,  0.0381, -0.0812,  0.0460],\n",
      "        ...,\n",
      "        [ 0.0648, -0.0006,  0.1119,  ...,  0.2001,  0.0187, -0.0690],\n",
      "        [-0.0545, -0.0381, -0.0122,  ..., -0.0547, -0.0565,  0.1097],\n",
      "        [-0.0096,  0.0585, -0.0793,  ..., -0.0826,  0.0140,  0.0393]])), ('model.decoder.layers.11.encoder_attn.out_proj.bias', tensor([-0.0392,  0.0283, -0.0328,  ...,  0.0284, -0.0082, -0.0554])), ('model.decoder.layers.11.encoder_attn_layer_norm.weight', tensor([0.8589, 0.9043, 0.8491,  ..., 0.8398, 0.8809, 0.7476])), ('model.decoder.layers.11.encoder_attn_layer_norm.bias', tensor([ 0.0443,  0.0036,  0.0288,  ...,  0.0290, -0.0180,  0.0395])), ('model.decoder.layers.11.fc1.weight', tensor([[ 2.7908e-02, -1.9263e-01, -3.0640e-02,  ...,  1.0760e-01,\n",
      "          8.9050e-02, -7.1533e-02],\n",
      "        [-4.1443e-02,  9.4727e-02, -5.5359e-02,  ...,  3.0197e-02,\n",
      "          1.0950e-01, -3.5858e-02],\n",
      "        [ 6.2988e-02,  3.7018e-02, -3.7048e-02,  ...,  8.5632e-02,\n",
      "         -6.4941e-02, -1.2158e-01],\n",
      "        ...,\n",
      "        [-2.4199e-05, -7.6141e-03, -1.1560e-01,  ..., -5.9448e-02,\n",
      "         -1.3220e-01, -3.3051e-02],\n",
      "        [ 1.8384e-01,  1.5649e-01, -1.0223e-02,  ...,  3.7201e-02,\n",
      "         -7.5264e-03,  1.1639e-01],\n",
      "        [ 4.7821e-02,  1.0596e-01, -2.9343e-02,  ..., -2.6172e-01,\n",
      "         -3.0472e-02,  7.9041e-02]])), ('model.decoder.layers.11.fc1.bias', tensor([-0.0572, -0.0735, -0.0285,  ..., -0.0360, -0.0496, -0.0848])), ('model.decoder.layers.11.fc2.weight', tensor([[-0.0125, -0.1775,  0.1017,  ...,  0.0686, -0.0693,  0.0669],\n",
      "        [-0.0704,  0.0434, -0.1445,  ...,  0.0336, -0.0676, -0.0312],\n",
      "        [-0.0390,  0.0815, -0.0345,  ...,  0.0110,  0.0038,  0.0005],\n",
      "        ...,\n",
      "        [-0.0018, -0.0315,  0.0848,  ..., -0.0508, -0.0586,  0.1174],\n",
      "        [-0.0130,  0.0298, -0.0439,  ..., -0.0124, -0.0767, -0.0703],\n",
      "        [-0.0258, -0.0396,  0.0561,  ..., -0.0724,  0.0035,  0.0140]])), ('model.decoder.layers.11.fc2.bias', tensor([ 0.0178,  0.1243, -0.0435,  ..., -0.0186, -0.0450, -0.0699])), ('model.decoder.layers.11.final_layer_norm.weight', tensor([0.7852, 0.8110, 0.8145,  ..., 0.8511, 0.8149, 0.8408])), ('model.decoder.layers.11.final_layer_norm.bias', tensor([ 0.0231, -0.0680,  0.0268,  ...,  0.0290,  0.0732,  0.0528])), ('model.decoder.layer_norm.weight', tensor([0.9980, 1.0000, 1.0000,  ..., 0.9995, 0.9990, 0.9990])), ('model.decoder.layer_norm.bias', tensor([-0.0402, -0.0060,  0.0126,  ..., -0.0626,  0.0779,  0.0125])), ('lm_head.weight', tensor([[ 0.0117, -0.0016, -0.0037,  ...,  0.0047, -0.0064, -0.0061],\n",
      "        [-0.0008, -0.0297,  0.0005,  ..., -0.0311,  0.0016, -0.0305],\n",
      "        [-0.0167,  0.0146, -0.0097,  ..., -0.0175,  0.0177, -0.0497],\n",
      "        ...,\n",
      "        [ 0.0285, -0.0268,  0.0322,  ...,  0.0152,  0.0374, -0.0690],\n",
      "        [-0.0064, -0.0241,  0.0508,  ..., -0.0005,  0.0751, -0.0540],\n",
      "        [ 0.0312,  0.0444,  0.0149,  ...,  0.0448,  0.0242, -0.0059]]))])\n"
     ]
    }
   ],
   "source": [
    "print(pruned_model.state_dict())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "97fbcfb1-e8c3-4634-9f44-fc3c609bfee7",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model.shared.weight MATCH!\n",
      "model.encoder.embed_positions.weight MATCH!\n",
      "model.encoder.layers.0.self_attn.k_proj.weight MATCH!\n",
      "model.encoder.layers.0.self_attn.k_proj.bias MATCH!\n",
      "model.encoder.layers.0.self_attn.v_proj.weight MATCH!\n",
      "model.encoder.layers.0.self_attn.v_proj.bias MATCH!\n",
      "model.encoder.layers.0.self_attn.q_proj.weight MATCH!\n",
      "model.encoder.layers.0.self_attn.q_proj.bias MATCH!\n",
      "model.encoder.layers.0.self_attn.out_proj.weight MATCH!\n",
      "model.encoder.layers.0.self_attn.out_proj.bias MATCH!\n",
      "model.encoder.layers.0.self_attn_layer_norm.weight MATCH!\n",
      "model.encoder.layers.0.self_attn_layer_norm.bias MATCH!\n",
      "model.encoder.layers.0.fc1.weight MATCH!\n",
      "model.encoder.layers.0.fc1.bias MATCH!\n",
      "model.encoder.layers.0.fc2.weight MATCH!\n",
      "model.encoder.layers.0.fc2.bias MATCH!\n",
      "model.encoder.layers.0.final_layer_norm.weight MATCH!\n",
      "model.encoder.layers.0.final_layer_norm.bias MATCH!\n",
      "model.encoder.layers.1.self_attn.k_proj.weight MATCH!\n",
      "model.encoder.layers.1.self_attn.k_proj.bias MATCH!\n",
      "model.encoder.layers.1.self_attn.v_proj.weight MATCH!\n",
      "model.encoder.layers.1.self_attn.v_proj.bias MATCH!\n",
      "model.encoder.layers.1.self_attn.q_proj.weight MATCH!\n",
      "model.encoder.layers.1.self_attn.q_proj.bias MATCH!\n",
      "model.encoder.layers.1.self_attn.out_proj.weight MATCH!\n",
      "model.encoder.layers.1.self_attn.out_proj.bias MATCH!\n",
      "model.encoder.layers.1.self_attn_layer_norm.weight MATCH!\n",
      "model.encoder.layers.1.self_attn_layer_norm.bias MATCH!\n",
      "model.encoder.layers.1.fc1.weight MATCH!\n",
      "model.encoder.layers.1.fc1.bias MATCH!\n",
      "model.encoder.layers.1.fc2.weight MATCH!\n",
      "model.encoder.layers.1.fc2.bias MATCH!\n",
      "model.encoder.layers.1.final_layer_norm.weight MATCH!\n",
      "model.encoder.layers.1.final_layer_norm.bias MATCH!\n",
      "model.encoder.layer_norm.weight MATCH!\n",
      "model.encoder.layer_norm.bias MATCH!\n",
      "model.decoder.embed_positions.weight MATCH!\n",
      "model.decoder.layers.0.self_attn.k_proj.weight MATCH!\n",
      "model.decoder.layers.0.self_attn.k_proj.bias MATCH!\n",
      "model.decoder.layers.0.self_attn.v_proj.weight MATCH!\n",
      "model.decoder.layers.0.self_attn.v_proj.bias MATCH!\n",
      "model.decoder.layers.0.self_attn.q_proj.weight MATCH!\n",
      "model.decoder.layers.0.self_attn.q_proj.bias MATCH!\n",
      "model.decoder.layers.0.self_attn.out_proj.weight MATCH!\n",
      "model.decoder.layers.0.self_attn.out_proj.bias MATCH!\n",
      "model.decoder.layers.0.self_attn_layer_norm.weight MATCH!\n",
      "model.decoder.layers.0.self_attn_layer_norm.bias MATCH!\n",
      "model.decoder.layers.0.encoder_attn.k_proj.weight MATCH!\n",
      "model.decoder.layers.0.encoder_attn.k_proj.bias MATCH!\n",
      "model.decoder.layers.0.encoder_attn.v_proj.weight MATCH!\n",
      "model.decoder.layers.0.encoder_attn.v_proj.bias MATCH!\n",
      "model.decoder.layers.0.encoder_attn.q_proj.weight MATCH!\n",
      "model.decoder.layers.0.encoder_attn.q_proj.bias MATCH!\n",
      "model.decoder.layers.0.encoder_attn.out_proj.weight MATCH!\n",
      "model.decoder.layers.0.encoder_attn.out_proj.bias MATCH!\n",
      "model.decoder.layers.0.encoder_attn_layer_norm.weight MATCH!\n",
      "model.decoder.layers.0.encoder_attn_layer_norm.bias MATCH!\n",
      "model.decoder.layers.0.fc1.weight MATCH!\n",
      "model.decoder.layers.0.fc1.bias MATCH!\n",
      "model.decoder.layers.0.fc2.weight MATCH!\n",
      "model.decoder.layers.0.fc2.bias MATCH!\n",
      "model.decoder.layers.0.final_layer_norm.weight MATCH!\n",
      "model.decoder.layers.0.final_layer_norm.bias MATCH!\n",
      "model.decoder.layers.1.self_attn.k_proj.weight MATCH!\n",
      "model.decoder.layers.1.self_attn.k_proj.bias MATCH!\n",
      "model.decoder.layers.1.self_attn.v_proj.weight MATCH!\n",
      "model.decoder.layers.1.self_attn.v_proj.bias MATCH!\n",
      "model.decoder.layers.1.self_attn.q_proj.weight MATCH!\n",
      "model.decoder.layers.1.self_attn.q_proj.bias MATCH!\n",
      "model.decoder.layers.1.self_attn.out_proj.weight MATCH!\n",
      "model.decoder.layers.1.self_attn.out_proj.bias MATCH!\n",
      "model.decoder.layers.1.self_attn_layer_norm.weight MATCH!\n",
      "model.decoder.layers.1.self_attn_layer_norm.bias MATCH!\n",
      "model.decoder.layers.1.encoder_attn.k_proj.weight MATCH!\n",
      "model.decoder.layers.1.encoder_attn.k_proj.bias MATCH!\n",
      "model.decoder.layers.1.encoder_attn.v_proj.weight MATCH!\n",
      "model.decoder.layers.1.encoder_attn.v_proj.bias MATCH!\n",
      "model.decoder.layers.1.encoder_attn.q_proj.weight MATCH!\n",
      "model.decoder.layers.1.encoder_attn.q_proj.bias MATCH!\n",
      "model.decoder.layers.1.encoder_attn.out_proj.weight MATCH!\n",
      "model.decoder.layers.1.encoder_attn.out_proj.bias MATCH!\n",
      "model.decoder.layers.1.encoder_attn_layer_norm.weight MATCH!\n",
      "model.decoder.layers.1.encoder_attn_layer_norm.bias MATCH!\n",
      "model.decoder.layers.1.fc1.weight MATCH!\n",
      "model.decoder.layers.1.fc1.bias MATCH!\n",
      "model.decoder.layers.1.fc2.weight MATCH!\n",
      "model.decoder.layers.1.fc2.bias MATCH!\n",
      "model.decoder.layers.1.final_layer_norm.weight MATCH!\n",
      "model.decoder.layers.1.final_layer_norm.bias MATCH!\n",
      "model.decoder.layers.2.self_attn.k_proj.weight MATCH!\n",
      "model.decoder.layers.2.self_attn.k_proj.bias MATCH!\n",
      "model.decoder.layers.2.self_attn.v_proj.weight MATCH!\n",
      "model.decoder.layers.2.self_attn.v_proj.bias MATCH!\n",
      "model.decoder.layers.2.self_attn.q_proj.weight MATCH!\n",
      "model.decoder.layers.2.self_attn.q_proj.bias MATCH!\n",
      "model.decoder.layers.2.self_attn.out_proj.weight MATCH!\n",
      "model.decoder.layers.2.self_attn.out_proj.bias MATCH!\n",
      "model.decoder.layers.2.self_attn_layer_norm.weight MATCH!\n",
      "model.decoder.layers.2.self_attn_layer_norm.bias MATCH!\n",
      "model.decoder.layers.2.encoder_attn.k_proj.weight MATCH!\n",
      "model.decoder.layers.2.encoder_attn.k_proj.bias MATCH!\n",
      "model.decoder.layers.2.encoder_attn.v_proj.weight MATCH!\n",
      "model.decoder.layers.2.encoder_attn.v_proj.bias MATCH!\n",
      "model.decoder.layers.2.encoder_attn.q_proj.weight MATCH!\n",
      "model.decoder.layers.2.encoder_attn.q_proj.bias MATCH!\n",
      "model.decoder.layers.2.encoder_attn.out_proj.weight MATCH!\n",
      "model.decoder.layers.2.encoder_attn.out_proj.bias MATCH!\n",
      "model.decoder.layers.2.encoder_attn_layer_norm.weight MATCH!\n",
      "model.decoder.layers.2.encoder_attn_layer_norm.bias MATCH!\n",
      "model.decoder.layers.2.fc1.weight MATCH!\n",
      "model.decoder.layers.2.fc1.bias MATCH!\n",
      "model.decoder.layers.2.fc2.weight MATCH!\n",
      "model.decoder.layers.2.fc2.bias MATCH!\n",
      "model.decoder.layers.2.final_layer_norm.weight MATCH!\n",
      "model.decoder.layers.2.final_layer_norm.bias MATCH!\n",
      "model.decoder.layers.3.self_attn.k_proj.weight MATCH!\n",
      "model.decoder.layers.3.self_attn.k_proj.bias MATCH!\n",
      "model.decoder.layers.3.self_attn.v_proj.weight MATCH!\n",
      "model.decoder.layers.3.self_attn.v_proj.bias MATCH!\n",
      "model.decoder.layers.3.self_attn.q_proj.weight MATCH!\n",
      "model.decoder.layers.3.self_attn.q_proj.bias MATCH!\n",
      "model.decoder.layers.3.self_attn.out_proj.weight MATCH!\n",
      "model.decoder.layers.3.self_attn.out_proj.bias MATCH!\n",
      "model.decoder.layers.3.self_attn_layer_norm.weight MATCH!\n",
      "model.decoder.layers.3.self_attn_layer_norm.bias MATCH!\n",
      "model.decoder.layers.3.encoder_attn.k_proj.weight MATCH!\n",
      "model.decoder.layers.3.encoder_attn.k_proj.bias MATCH!\n",
      "model.decoder.layers.3.encoder_attn.v_proj.weight MATCH!\n",
      "model.decoder.layers.3.encoder_attn.v_proj.bias MATCH!\n",
      "model.decoder.layers.3.encoder_attn.q_proj.weight MATCH!\n",
      "model.decoder.layers.3.encoder_attn.q_proj.bias MATCH!\n",
      "model.decoder.layers.3.encoder_attn.out_proj.weight MATCH!\n",
      "model.decoder.layers.3.encoder_attn.out_proj.bias MATCH!\n",
      "model.decoder.layers.3.encoder_attn_layer_norm.weight MATCH!\n",
      "model.decoder.layers.3.encoder_attn_layer_norm.bias MATCH!\n",
      "model.decoder.layers.3.fc1.weight MATCH!\n",
      "model.decoder.layers.3.fc1.bias MATCH!\n",
      "model.decoder.layers.3.fc2.weight MATCH!\n",
      "model.decoder.layers.3.fc2.bias MATCH!\n",
      "model.decoder.layers.3.final_layer_norm.weight MATCH!\n",
      "model.decoder.layers.3.final_layer_norm.bias MATCH!\n",
      "model.decoder.layers.4.self_attn.k_proj.weight MATCH!\n",
      "model.decoder.layers.4.self_attn.k_proj.bias MATCH!\n",
      "model.decoder.layers.4.self_attn.v_proj.weight MATCH!\n",
      "model.decoder.layers.4.self_attn.v_proj.bias MATCH!\n",
      "model.decoder.layers.4.self_attn.q_proj.weight MATCH!\n",
      "model.decoder.layers.4.self_attn.q_proj.bias MATCH!\n",
      "model.decoder.layers.4.self_attn.out_proj.weight MATCH!\n",
      "model.decoder.layers.4.self_attn.out_proj.bias MATCH!\n",
      "model.decoder.layers.4.self_attn_layer_norm.weight MATCH!\n",
      "model.decoder.layers.4.self_attn_layer_norm.bias MATCH!\n",
      "model.decoder.layers.4.encoder_attn.k_proj.weight MATCH!\n",
      "model.decoder.layers.4.encoder_attn.k_proj.bias MATCH!\n",
      "model.decoder.layers.4.encoder_attn.v_proj.weight MATCH!\n",
      "model.decoder.layers.4.encoder_attn.v_proj.bias MATCH!\n",
      "model.decoder.layers.4.encoder_attn.q_proj.weight MATCH!\n",
      "model.decoder.layers.4.encoder_attn.q_proj.bias MATCH!\n",
      "model.decoder.layers.4.encoder_attn.out_proj.weight MATCH!\n",
      "model.decoder.layers.4.encoder_attn.out_proj.bias MATCH!\n",
      "model.decoder.layers.4.encoder_attn_layer_norm.weight MATCH!\n",
      "model.decoder.layers.4.encoder_attn_layer_norm.bias MATCH!\n",
      "model.decoder.layers.4.fc1.weight MATCH!\n",
      "model.decoder.layers.4.fc1.bias MATCH!\n",
      "model.decoder.layers.4.fc2.weight MATCH!\n",
      "model.decoder.layers.4.fc2.bias MATCH!\n",
      "model.decoder.layers.4.final_layer_norm.weight MATCH!\n",
      "model.decoder.layers.4.final_layer_norm.bias MATCH!\n",
      "model.decoder.layers.5.self_attn.k_proj.weight MATCH!\n",
      "model.decoder.layers.5.self_attn.k_proj.bias MATCH!\n",
      "model.decoder.layers.5.self_attn.v_proj.weight MATCH!\n",
      "model.decoder.layers.5.self_attn.v_proj.bias MATCH!\n",
      "model.decoder.layers.5.self_attn.q_proj.weight MATCH!\n",
      "model.decoder.layers.5.self_attn.q_proj.bias MATCH!\n",
      "model.decoder.layers.5.self_attn.out_proj.weight MATCH!\n",
      "model.decoder.layers.5.self_attn.out_proj.bias MATCH!\n",
      "model.decoder.layers.5.self_attn_layer_norm.weight MATCH!\n",
      "model.decoder.layers.5.self_attn_layer_norm.bias MATCH!\n",
      "model.decoder.layers.5.encoder_attn.k_proj.weight MATCH!\n",
      "model.decoder.layers.5.encoder_attn.k_proj.bias MATCH!\n",
      "model.decoder.layers.5.encoder_attn.v_proj.weight MATCH!\n",
      "model.decoder.layers.5.encoder_attn.v_proj.bias MATCH!\n",
      "model.decoder.layers.5.encoder_attn.q_proj.weight MATCH!\n",
      "model.decoder.layers.5.encoder_attn.q_proj.bias MATCH!\n",
      "model.decoder.layers.5.encoder_attn.out_proj.weight MATCH!\n",
      "model.decoder.layers.5.encoder_attn.out_proj.bias MATCH!\n",
      "model.decoder.layers.5.encoder_attn_layer_norm.weight MATCH!\n",
      "model.decoder.layers.5.encoder_attn_layer_norm.bias MATCH!\n",
      "model.decoder.layers.5.fc1.weight MATCH!\n",
      "model.decoder.layers.5.fc1.bias MATCH!\n",
      "model.decoder.layers.5.fc2.weight MATCH!\n",
      "model.decoder.layers.5.fc2.bias MATCH!\n",
      "model.decoder.layers.5.final_layer_norm.weight MATCH!\n",
      "model.decoder.layers.5.final_layer_norm.bias MATCH!\n",
      "model.decoder.layers.6.self_attn.k_proj.weight MATCH!\n",
      "model.decoder.layers.6.self_attn.k_proj.bias MATCH!\n",
      "model.decoder.layers.6.self_attn.v_proj.weight MATCH!\n",
      "model.decoder.layers.6.self_attn.v_proj.bias MATCH!\n",
      "model.decoder.layers.6.self_attn.q_proj.weight MATCH!\n",
      "model.decoder.layers.6.self_attn.q_proj.bias MATCH!\n",
      "model.decoder.layers.6.self_attn.out_proj.weight MATCH!\n",
      "model.decoder.layers.6.self_attn.out_proj.bias MATCH!\n",
      "model.decoder.layers.6.self_attn_layer_norm.weight MATCH!\n",
      "model.decoder.layers.6.self_attn_layer_norm.bias MATCH!\n",
      "model.decoder.layers.6.encoder_attn.k_proj.weight MATCH!\n",
      "model.decoder.layers.6.encoder_attn.k_proj.bias MATCH!\n",
      "model.decoder.layers.6.encoder_attn.v_proj.weight MATCH!\n",
      "model.decoder.layers.6.encoder_attn.v_proj.bias MATCH!\n",
      "model.decoder.layers.6.encoder_attn.q_proj.weight MATCH!\n",
      "model.decoder.layers.6.encoder_attn.q_proj.bias MATCH!\n",
      "model.decoder.layers.6.encoder_attn.out_proj.weight MATCH!\n",
      "model.decoder.layers.6.encoder_attn.out_proj.bias MATCH!\n",
      "model.decoder.layers.6.encoder_attn_layer_norm.weight MATCH!\n",
      "model.decoder.layers.6.encoder_attn_layer_norm.bias MATCH!\n",
      "model.decoder.layers.6.fc1.weight MATCH!\n",
      "model.decoder.layers.6.fc1.bias MATCH!\n",
      "model.decoder.layers.6.fc2.weight MATCH!\n",
      "model.decoder.layers.6.fc2.bias MATCH!\n",
      "model.decoder.layers.6.final_layer_norm.weight MATCH!\n",
      "model.decoder.layers.6.final_layer_norm.bias MATCH!\n",
      "model.decoder.layers.7.self_attn.k_proj.weight MATCH!\n",
      "model.decoder.layers.7.self_attn.k_proj.bias MATCH!\n",
      "model.decoder.layers.7.self_attn.v_proj.weight MATCH!\n",
      "model.decoder.layers.7.self_attn.v_proj.bias MATCH!\n",
      "model.decoder.layers.7.self_attn.q_proj.weight MATCH!\n",
      "model.decoder.layers.7.self_attn.q_proj.bias MATCH!\n",
      "model.decoder.layers.7.self_attn.out_proj.weight MATCH!\n",
      "model.decoder.layers.7.self_attn.out_proj.bias MATCH!\n",
      "model.decoder.layers.7.self_attn_layer_norm.weight MATCH!\n",
      "model.decoder.layers.7.self_attn_layer_norm.bias MATCH!\n",
      "model.decoder.layers.7.encoder_attn.k_proj.weight MATCH!\n",
      "model.decoder.layers.7.encoder_attn.k_proj.bias MATCH!\n",
      "model.decoder.layers.7.encoder_attn.v_proj.weight MATCH!\n",
      "model.decoder.layers.7.encoder_attn.v_proj.bias MATCH!\n",
      "model.decoder.layers.7.encoder_attn.q_proj.weight MATCH!\n",
      "model.decoder.layers.7.encoder_attn.q_proj.bias MATCH!\n",
      "model.decoder.layers.7.encoder_attn.out_proj.weight MATCH!\n",
      "model.decoder.layers.7.encoder_attn.out_proj.bias MATCH!\n",
      "model.decoder.layers.7.encoder_attn_layer_norm.weight MATCH!\n",
      "model.decoder.layers.7.encoder_attn_layer_norm.bias MATCH!\n",
      "model.decoder.layers.7.fc1.weight MATCH!\n",
      "model.decoder.layers.7.fc1.bias MATCH!\n",
      "model.decoder.layers.7.fc2.weight MATCH!\n",
      "model.decoder.layers.7.fc2.bias MATCH!\n",
      "model.decoder.layers.7.final_layer_norm.weight MATCH!\n",
      "model.decoder.layers.7.final_layer_norm.bias MATCH!\n",
      "model.decoder.layers.8.self_attn.k_proj.weight MATCH!\n",
      "model.decoder.layers.8.self_attn.k_proj.bias MATCH!\n",
      "model.decoder.layers.8.self_attn.v_proj.weight MATCH!\n",
      "model.decoder.layers.8.self_attn.v_proj.bias MATCH!\n",
      "model.decoder.layers.8.self_attn.q_proj.weight MATCH!\n",
      "model.decoder.layers.8.self_attn.q_proj.bias MATCH!\n",
      "model.decoder.layers.8.self_attn.out_proj.weight MATCH!\n",
      "model.decoder.layers.8.self_attn.out_proj.bias MATCH!\n",
      "model.decoder.layers.8.self_attn_layer_norm.weight MATCH!\n",
      "model.decoder.layers.8.self_attn_layer_norm.bias MATCH!\n",
      "model.decoder.layers.8.encoder_attn.k_proj.weight MATCH!\n",
      "model.decoder.layers.8.encoder_attn.k_proj.bias MATCH!\n",
      "model.decoder.layers.8.encoder_attn.v_proj.weight MATCH!\n",
      "model.decoder.layers.8.encoder_attn.v_proj.bias MATCH!\n",
      "model.decoder.layers.8.encoder_attn.q_proj.weight MATCH!\n",
      "model.decoder.layers.8.encoder_attn.q_proj.bias MATCH!\n",
      "model.decoder.layers.8.encoder_attn.out_proj.weight MATCH!\n",
      "model.decoder.layers.8.encoder_attn.out_proj.bias MATCH!\n",
      "model.decoder.layers.8.encoder_attn_layer_norm.weight MATCH!\n",
      "model.decoder.layers.8.encoder_attn_layer_norm.bias MATCH!\n",
      "model.decoder.layers.8.fc1.weight MATCH!\n",
      "model.decoder.layers.8.fc1.bias MATCH!\n",
      "model.decoder.layers.8.fc2.weight MATCH!\n",
      "model.decoder.layers.8.fc2.bias MATCH!\n",
      "model.decoder.layers.8.final_layer_norm.weight MATCH!\n",
      "model.decoder.layers.8.final_layer_norm.bias MATCH!\n",
      "model.decoder.layers.9.self_attn.k_proj.weight MATCH!\n",
      "model.decoder.layers.9.self_attn.k_proj.bias MATCH!\n",
      "model.decoder.layers.9.self_attn.v_proj.weight MATCH!\n",
      "model.decoder.layers.9.self_attn.v_proj.bias MATCH!\n",
      "model.decoder.layers.9.self_attn.q_proj.weight MATCH!\n",
      "model.decoder.layers.9.self_attn.q_proj.bias MATCH!\n",
      "model.decoder.layers.9.self_attn.out_proj.weight MATCH!\n",
      "model.decoder.layers.9.self_attn.out_proj.bias MATCH!\n",
      "model.decoder.layers.9.self_attn_layer_norm.weight MATCH!\n",
      "model.decoder.layers.9.self_attn_layer_norm.bias MATCH!\n",
      "model.decoder.layers.9.encoder_attn.k_proj.weight MATCH!\n",
      "model.decoder.layers.9.encoder_attn.k_proj.bias MATCH!\n",
      "model.decoder.layers.9.encoder_attn.v_proj.weight MATCH!\n",
      "model.decoder.layers.9.encoder_attn.v_proj.bias MATCH!\n",
      "model.decoder.layers.9.encoder_attn.q_proj.weight MATCH!\n",
      "model.decoder.layers.9.encoder_attn.q_proj.bias MATCH!\n",
      "model.decoder.layers.9.encoder_attn.out_proj.weight MATCH!\n",
      "model.decoder.layers.9.encoder_attn.out_proj.bias MATCH!\n",
      "model.decoder.layers.9.encoder_attn_layer_norm.weight MATCH!\n",
      "model.decoder.layers.9.encoder_attn_layer_norm.bias MATCH!\n",
      "model.decoder.layers.9.fc1.weight MATCH!\n",
      "model.decoder.layers.9.fc1.bias MATCH!\n",
      "model.decoder.layers.9.fc2.weight MATCH!\n",
      "model.decoder.layers.9.fc2.bias MATCH!\n",
      "model.decoder.layers.9.final_layer_norm.weight MATCH!\n",
      "model.decoder.layers.9.final_layer_norm.bias MATCH!\n",
      "model.decoder.layers.10.self_attn.k_proj.weight MATCH!\n",
      "model.decoder.layers.10.self_attn.k_proj.bias MATCH!\n",
      "model.decoder.layers.10.self_attn.v_proj.weight MATCH!\n",
      "model.decoder.layers.10.self_attn.v_proj.bias MATCH!\n",
      "model.decoder.layers.10.self_attn.q_proj.weight MATCH!\n",
      "model.decoder.layers.10.self_attn.q_proj.bias MATCH!\n",
      "model.decoder.layers.10.self_attn.out_proj.weight MATCH!\n",
      "model.decoder.layers.10.self_attn.out_proj.bias MATCH!\n",
      "model.decoder.layers.10.self_attn_layer_norm.weight MATCH!\n",
      "model.decoder.layers.10.self_attn_layer_norm.bias MATCH!\n",
      "model.decoder.layers.10.encoder_attn.k_proj.weight MATCH!\n",
      "model.decoder.layers.10.encoder_attn.k_proj.bias MATCH!\n",
      "model.decoder.layers.10.encoder_attn.v_proj.weight MATCH!\n",
      "model.decoder.layers.10.encoder_attn.v_proj.bias MATCH!\n",
      "model.decoder.layers.10.encoder_attn.q_proj.weight MATCH!\n",
      "model.decoder.layers.10.encoder_attn.q_proj.bias MATCH!\n",
      "model.decoder.layers.10.encoder_attn.out_proj.weight MATCH!\n",
      "model.decoder.layers.10.encoder_attn.out_proj.bias MATCH!\n",
      "model.decoder.layers.10.encoder_attn_layer_norm.weight MATCH!\n",
      "model.decoder.layers.10.encoder_attn_layer_norm.bias MATCH!\n",
      "model.decoder.layers.10.fc1.weight MATCH!\n",
      "model.decoder.layers.10.fc1.bias MATCH!\n",
      "model.decoder.layers.10.fc2.weight MATCH!\n",
      "model.decoder.layers.10.fc2.bias MATCH!\n",
      "model.decoder.layers.10.final_layer_norm.weight MATCH!\n",
      "model.decoder.layers.10.final_layer_norm.bias MATCH!\n",
      "model.decoder.layers.11.self_attn.k_proj.weight MATCH!\n",
      "model.decoder.layers.11.self_attn.k_proj.bias MATCH!\n",
      "model.decoder.layers.11.self_attn.v_proj.weight MATCH!\n",
      "model.decoder.layers.11.self_attn.v_proj.bias MATCH!\n",
      "model.decoder.layers.11.self_attn.q_proj.weight MATCH!\n",
      "model.decoder.layers.11.self_attn.q_proj.bias MATCH!\n",
      "model.decoder.layers.11.self_attn.out_proj.weight MATCH!\n",
      "model.decoder.layers.11.self_attn.out_proj.bias MATCH!\n",
      "model.decoder.layers.11.self_attn_layer_norm.weight MATCH!\n",
      "model.decoder.layers.11.self_attn_layer_norm.bias MATCH!\n",
      "model.decoder.layers.11.encoder_attn.k_proj.weight MATCH!\n",
      "model.decoder.layers.11.encoder_attn.k_proj.bias MATCH!\n",
      "model.decoder.layers.11.encoder_attn.v_proj.weight MATCH!\n",
      "model.decoder.layers.11.encoder_attn.v_proj.bias MATCH!\n",
      "model.decoder.layers.11.encoder_attn.q_proj.weight MATCH!\n",
      "model.decoder.layers.11.encoder_attn.q_proj.bias MATCH!\n",
      "model.decoder.layers.11.encoder_attn.out_proj.weight MATCH!\n",
      "model.decoder.layers.11.encoder_attn.out_proj.bias MATCH!\n",
      "model.decoder.layers.11.encoder_attn_layer_norm.weight MATCH!\n",
      "model.decoder.layers.11.encoder_attn_layer_norm.bias MATCH!\n",
      "model.decoder.layers.11.fc1.weight MATCH!\n",
      "model.decoder.layers.11.fc1.bias MATCH!\n",
      "model.decoder.layers.11.fc2.weight MATCH!\n",
      "model.decoder.layers.11.fc2.bias MATCH!\n",
      "model.decoder.layers.11.final_layer_norm.weight MATCH!\n",
      "model.decoder.layers.11.final_layer_norm.bias MATCH!\n",
      "model.decoder.layer_norm.weight MATCH!\n",
      "model.decoder.layer_norm.bias MATCH!\n"
     ]
    }
   ],
   "source": [
    "for name, weight in model.named_parameters():\n",
    "    if torch.all(torch.eq(weight, pruned_model.state_dict()[name])):\n",
    "        print(name, \"MATCH!\")\n",
    "    else:\n",
    "        print(name, \"NOOOOPE\")\n",
    "    # print(name, weight)\n",
    "    # print(pruned_model.state_dict()[name])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "da4f3dfb-8d42-46b4-98bd-887e14b4c329",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_statedict = model.state_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "1ffea695-61b9-4e4a-951f-cb6b301b8adf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "final_logits_bias\n",
      "model.shared.weight\n",
      "model.encoder.embed_tokens.weight\n",
      "model.encoder.embed_positions.weight\n",
      "model.encoder.layers.0.self_attn.k_proj.weight\n",
      "model.encoder.layers.0.self_attn.k_proj.bias\n",
      "model.encoder.layers.0.self_attn.v_proj.weight\n",
      "model.encoder.layers.0.self_attn.v_proj.bias\n",
      "model.encoder.layers.0.self_attn.q_proj.weight\n",
      "model.encoder.layers.0.self_attn.q_proj.bias\n",
      "model.encoder.layers.0.self_attn.out_proj.weight\n",
      "model.encoder.layers.0.self_attn.out_proj.bias\n",
      "model.encoder.layers.0.self_attn_layer_norm.weight\n",
      "model.encoder.layers.0.self_attn_layer_norm.bias\n",
      "model.encoder.layers.0.fc1.weight\n",
      "model.encoder.layers.0.fc1.bias\n",
      "model.encoder.layers.0.fc2.weight\n",
      "model.encoder.layers.0.fc2.bias\n",
      "model.encoder.layers.0.final_layer_norm.weight\n",
      "model.encoder.layers.0.final_layer_norm.bias\n",
      "model.encoder.layers.1.self_attn.k_proj.weight\n",
      "model.encoder.layers.1.self_attn.k_proj.bias\n",
      "model.encoder.layers.1.self_attn.v_proj.weight\n",
      "model.encoder.layers.1.self_attn.v_proj.bias\n",
      "model.encoder.layers.1.self_attn.q_proj.weight\n",
      "model.encoder.layers.1.self_attn.q_proj.bias\n",
      "model.encoder.layers.1.self_attn.out_proj.weight\n",
      "model.encoder.layers.1.self_attn.out_proj.bias\n",
      "model.encoder.layers.1.self_attn_layer_norm.weight\n",
      "model.encoder.layers.1.self_attn_layer_norm.bias\n",
      "model.encoder.layers.1.fc1.weight\n",
      "model.encoder.layers.1.fc1.bias\n",
      "model.encoder.layers.1.fc2.weight\n",
      "model.encoder.layers.1.fc2.bias\n",
      "model.encoder.layers.1.final_layer_norm.weight\n",
      "model.encoder.layers.1.final_layer_norm.bias\n",
      "model.encoder.layer_norm.weight\n",
      "model.encoder.layer_norm.bias\n",
      "model.decoder.embed_tokens.weight\n",
      "model.decoder.embed_positions.weight\n",
      "model.decoder.layers.0.self_attn.k_proj.weight\n",
      "model.decoder.layers.0.self_attn.k_proj.bias\n",
      "model.decoder.layers.0.self_attn.v_proj.weight\n",
      "model.decoder.layers.0.self_attn.v_proj.bias\n",
      "model.decoder.layers.0.self_attn.q_proj.weight\n",
      "model.decoder.layers.0.self_attn.q_proj.bias\n",
      "model.decoder.layers.0.self_attn.out_proj.weight\n",
      "model.decoder.layers.0.self_attn.out_proj.bias\n",
      "model.decoder.layers.0.self_attn_layer_norm.weight\n",
      "model.decoder.layers.0.self_attn_layer_norm.bias\n",
      "model.decoder.layers.0.encoder_attn.k_proj.weight\n",
      "model.decoder.layers.0.encoder_attn.k_proj.bias\n",
      "model.decoder.layers.0.encoder_attn.v_proj.weight\n",
      "model.decoder.layers.0.encoder_attn.v_proj.bias\n",
      "model.decoder.layers.0.encoder_attn.q_proj.weight\n",
      "model.decoder.layers.0.encoder_attn.q_proj.bias\n",
      "model.decoder.layers.0.encoder_attn.out_proj.weight\n",
      "model.decoder.layers.0.encoder_attn.out_proj.bias\n",
      "model.decoder.layers.0.encoder_attn_layer_norm.weight\n",
      "model.decoder.layers.0.encoder_attn_layer_norm.bias\n",
      "model.decoder.layers.0.fc1.weight\n",
      "model.decoder.layers.0.fc1.bias\n",
      "model.decoder.layers.0.fc2.weight\n",
      "model.decoder.layers.0.fc2.bias\n",
      "model.decoder.layers.0.final_layer_norm.weight\n",
      "model.decoder.layers.0.final_layer_norm.bias\n",
      "model.decoder.layers.1.self_attn.k_proj.weight\n",
      "model.decoder.layers.1.self_attn.k_proj.bias\n",
      "model.decoder.layers.1.self_attn.v_proj.weight\n",
      "model.decoder.layers.1.self_attn.v_proj.bias\n",
      "model.decoder.layers.1.self_attn.q_proj.weight\n",
      "model.decoder.layers.1.self_attn.q_proj.bias\n",
      "model.decoder.layers.1.self_attn.out_proj.weight\n",
      "model.decoder.layers.1.self_attn.out_proj.bias\n",
      "model.decoder.layers.1.self_attn_layer_norm.weight\n",
      "model.decoder.layers.1.self_attn_layer_norm.bias\n",
      "model.decoder.layers.1.encoder_attn.k_proj.weight\n",
      "model.decoder.layers.1.encoder_attn.k_proj.bias\n",
      "model.decoder.layers.1.encoder_attn.v_proj.weight\n",
      "model.decoder.layers.1.encoder_attn.v_proj.bias\n",
      "model.decoder.layers.1.encoder_attn.q_proj.weight\n",
      "model.decoder.layers.1.encoder_attn.q_proj.bias\n",
      "model.decoder.layers.1.encoder_attn.out_proj.weight\n",
      "model.decoder.layers.1.encoder_attn.out_proj.bias\n",
      "model.decoder.layers.1.encoder_attn_layer_norm.weight\n",
      "model.decoder.layers.1.encoder_attn_layer_norm.bias\n",
      "model.decoder.layers.1.fc1.weight\n",
      "model.decoder.layers.1.fc1.bias\n",
      "model.decoder.layers.1.fc2.weight\n",
      "model.decoder.layers.1.fc2.bias\n",
      "model.decoder.layers.1.final_layer_norm.weight\n",
      "model.decoder.layers.1.final_layer_norm.bias\n",
      "model.decoder.layers.2.self_attn.k_proj.weight\n",
      "model.decoder.layers.2.self_attn.k_proj.bias\n",
      "model.decoder.layers.2.self_attn.v_proj.weight\n",
      "model.decoder.layers.2.self_attn.v_proj.bias\n",
      "model.decoder.layers.2.self_attn.q_proj.weight\n",
      "model.decoder.layers.2.self_attn.q_proj.bias\n",
      "model.decoder.layers.2.self_attn.out_proj.weight\n",
      "model.decoder.layers.2.self_attn.out_proj.bias\n",
      "model.decoder.layers.2.self_attn_layer_norm.weight\n",
      "model.decoder.layers.2.self_attn_layer_norm.bias\n",
      "model.decoder.layers.2.encoder_attn.k_proj.weight\n",
      "model.decoder.layers.2.encoder_attn.k_proj.bias\n",
      "model.decoder.layers.2.encoder_attn.v_proj.weight\n",
      "model.decoder.layers.2.encoder_attn.v_proj.bias\n",
      "model.decoder.layers.2.encoder_attn.q_proj.weight\n",
      "model.decoder.layers.2.encoder_attn.q_proj.bias\n",
      "model.decoder.layers.2.encoder_attn.out_proj.weight\n",
      "model.decoder.layers.2.encoder_attn.out_proj.bias\n",
      "model.decoder.layers.2.encoder_attn_layer_norm.weight\n",
      "model.decoder.layers.2.encoder_attn_layer_norm.bias\n",
      "model.decoder.layers.2.fc1.weight\n",
      "model.decoder.layers.2.fc1.bias\n",
      "model.decoder.layers.2.fc2.weight\n",
      "model.decoder.layers.2.fc2.bias\n",
      "model.decoder.layers.2.final_layer_norm.weight\n",
      "model.decoder.layers.2.final_layer_norm.bias\n",
      "model.decoder.layers.3.self_attn.k_proj.weight\n",
      "model.decoder.layers.3.self_attn.k_proj.bias\n",
      "model.decoder.layers.3.self_attn.v_proj.weight\n",
      "model.decoder.layers.3.self_attn.v_proj.bias\n",
      "model.decoder.layers.3.self_attn.q_proj.weight\n",
      "model.decoder.layers.3.self_attn.q_proj.bias\n",
      "model.decoder.layers.3.self_attn.out_proj.weight\n",
      "model.decoder.layers.3.self_attn.out_proj.bias\n",
      "model.decoder.layers.3.self_attn_layer_norm.weight\n",
      "model.decoder.layers.3.self_attn_layer_norm.bias\n",
      "model.decoder.layers.3.encoder_attn.k_proj.weight\n",
      "model.decoder.layers.3.encoder_attn.k_proj.bias\n",
      "model.decoder.layers.3.encoder_attn.v_proj.weight\n",
      "model.decoder.layers.3.encoder_attn.v_proj.bias\n",
      "model.decoder.layers.3.encoder_attn.q_proj.weight\n",
      "model.decoder.layers.3.encoder_attn.q_proj.bias\n",
      "model.decoder.layers.3.encoder_attn.out_proj.weight\n",
      "model.decoder.layers.3.encoder_attn.out_proj.bias\n",
      "model.decoder.layers.3.encoder_attn_layer_norm.weight\n",
      "model.decoder.layers.3.encoder_attn_layer_norm.bias\n",
      "model.decoder.layers.3.fc1.weight\n",
      "model.decoder.layers.3.fc1.bias\n",
      "model.decoder.layers.3.fc2.weight\n",
      "model.decoder.layers.3.fc2.bias\n",
      "model.decoder.layers.3.final_layer_norm.weight\n",
      "model.decoder.layers.3.final_layer_norm.bias\n",
      "model.decoder.layers.4.self_attn.k_proj.weight\n",
      "model.decoder.layers.4.self_attn.k_proj.bias\n",
      "model.decoder.layers.4.self_attn.v_proj.weight\n",
      "model.decoder.layers.4.self_attn.v_proj.bias\n",
      "model.decoder.layers.4.self_attn.q_proj.weight\n",
      "model.decoder.layers.4.self_attn.q_proj.bias\n",
      "model.decoder.layers.4.self_attn.out_proj.weight\n",
      "model.decoder.layers.4.self_attn.out_proj.bias\n",
      "model.decoder.layers.4.self_attn_layer_norm.weight\n",
      "model.decoder.layers.4.self_attn_layer_norm.bias\n",
      "model.decoder.layers.4.encoder_attn.k_proj.weight\n",
      "model.decoder.layers.4.encoder_attn.k_proj.bias\n",
      "model.decoder.layers.4.encoder_attn.v_proj.weight\n",
      "model.decoder.layers.4.encoder_attn.v_proj.bias\n",
      "model.decoder.layers.4.encoder_attn.q_proj.weight\n",
      "model.decoder.layers.4.encoder_attn.q_proj.bias\n",
      "model.decoder.layers.4.encoder_attn.out_proj.weight\n",
      "model.decoder.layers.4.encoder_attn.out_proj.bias\n",
      "model.decoder.layers.4.encoder_attn_layer_norm.weight\n",
      "model.decoder.layers.4.encoder_attn_layer_norm.bias\n",
      "model.decoder.layers.4.fc1.weight\n",
      "model.decoder.layers.4.fc1.bias\n",
      "model.decoder.layers.4.fc2.weight\n",
      "model.decoder.layers.4.fc2.bias\n",
      "model.decoder.layers.4.final_layer_norm.weight\n",
      "model.decoder.layers.4.final_layer_norm.bias\n",
      "model.decoder.layers.5.self_attn.k_proj.weight\n",
      "model.decoder.layers.5.self_attn.k_proj.bias\n",
      "model.decoder.layers.5.self_attn.v_proj.weight\n",
      "model.decoder.layers.5.self_attn.v_proj.bias\n",
      "model.decoder.layers.5.self_attn.q_proj.weight\n",
      "model.decoder.layers.5.self_attn.q_proj.bias\n",
      "model.decoder.layers.5.self_attn.out_proj.weight\n",
      "model.decoder.layers.5.self_attn.out_proj.bias\n",
      "model.decoder.layers.5.self_attn_layer_norm.weight\n",
      "model.decoder.layers.5.self_attn_layer_norm.bias\n",
      "model.decoder.layers.5.encoder_attn.k_proj.weight\n",
      "model.decoder.layers.5.encoder_attn.k_proj.bias\n",
      "model.decoder.layers.5.encoder_attn.v_proj.weight\n",
      "model.decoder.layers.5.encoder_attn.v_proj.bias\n",
      "model.decoder.layers.5.encoder_attn.q_proj.weight\n",
      "model.decoder.layers.5.encoder_attn.q_proj.bias\n",
      "model.decoder.layers.5.encoder_attn.out_proj.weight\n",
      "model.decoder.layers.5.encoder_attn.out_proj.bias\n",
      "model.decoder.layers.5.encoder_attn_layer_norm.weight\n",
      "model.decoder.layers.5.encoder_attn_layer_norm.bias\n",
      "model.decoder.layers.5.fc1.weight\n",
      "model.decoder.layers.5.fc1.bias\n",
      "model.decoder.layers.5.fc2.weight\n",
      "model.decoder.layers.5.fc2.bias\n",
      "model.decoder.layers.5.final_layer_norm.weight\n",
      "model.decoder.layers.5.final_layer_norm.bias\n",
      "model.decoder.layers.6.self_attn.k_proj.weight\n",
      "model.decoder.layers.6.self_attn.k_proj.bias\n",
      "model.decoder.layers.6.self_attn.v_proj.weight\n",
      "model.decoder.layers.6.self_attn.v_proj.bias\n",
      "model.decoder.layers.6.self_attn.q_proj.weight\n",
      "model.decoder.layers.6.self_attn.q_proj.bias\n",
      "model.decoder.layers.6.self_attn.out_proj.weight\n",
      "model.decoder.layers.6.self_attn.out_proj.bias\n",
      "model.decoder.layers.6.self_attn_layer_norm.weight\n",
      "model.decoder.layers.6.self_attn_layer_norm.bias\n",
      "model.decoder.layers.6.encoder_attn.k_proj.weight\n",
      "model.decoder.layers.6.encoder_attn.k_proj.bias\n",
      "model.decoder.layers.6.encoder_attn.v_proj.weight\n",
      "model.decoder.layers.6.encoder_attn.v_proj.bias\n",
      "model.decoder.layers.6.encoder_attn.q_proj.weight\n",
      "model.decoder.layers.6.encoder_attn.q_proj.bias\n",
      "model.decoder.layers.6.encoder_attn.out_proj.weight\n",
      "model.decoder.layers.6.encoder_attn.out_proj.bias\n",
      "model.decoder.layers.6.encoder_attn_layer_norm.weight\n",
      "model.decoder.layers.6.encoder_attn_layer_norm.bias\n",
      "model.decoder.layers.6.fc1.weight\n",
      "model.decoder.layers.6.fc1.bias\n",
      "model.decoder.layers.6.fc2.weight\n",
      "model.decoder.layers.6.fc2.bias\n",
      "model.decoder.layers.6.final_layer_norm.weight\n",
      "model.decoder.layers.6.final_layer_norm.bias\n",
      "model.decoder.layers.7.self_attn.k_proj.weight\n",
      "model.decoder.layers.7.self_attn.k_proj.bias\n",
      "model.decoder.layers.7.self_attn.v_proj.weight\n",
      "model.decoder.layers.7.self_attn.v_proj.bias\n",
      "model.decoder.layers.7.self_attn.q_proj.weight\n",
      "model.decoder.layers.7.self_attn.q_proj.bias\n",
      "model.decoder.layers.7.self_attn.out_proj.weight\n",
      "model.decoder.layers.7.self_attn.out_proj.bias\n",
      "model.decoder.layers.7.self_attn_layer_norm.weight\n",
      "model.decoder.layers.7.self_attn_layer_norm.bias\n",
      "model.decoder.layers.7.encoder_attn.k_proj.weight\n",
      "model.decoder.layers.7.encoder_attn.k_proj.bias\n",
      "model.decoder.layers.7.encoder_attn.v_proj.weight\n",
      "model.decoder.layers.7.encoder_attn.v_proj.bias\n",
      "model.decoder.layers.7.encoder_attn.q_proj.weight\n",
      "model.decoder.layers.7.encoder_attn.q_proj.bias\n",
      "model.decoder.layers.7.encoder_attn.out_proj.weight\n",
      "model.decoder.layers.7.encoder_attn.out_proj.bias\n",
      "model.decoder.layers.7.encoder_attn_layer_norm.weight\n",
      "model.decoder.layers.7.encoder_attn_layer_norm.bias\n",
      "model.decoder.layers.7.fc1.weight\n",
      "model.decoder.layers.7.fc1.bias\n",
      "model.decoder.layers.7.fc2.weight\n",
      "model.decoder.layers.7.fc2.bias\n",
      "model.decoder.layers.7.final_layer_norm.weight\n",
      "model.decoder.layers.7.final_layer_norm.bias\n",
      "model.decoder.layers.8.self_attn.k_proj.weight\n",
      "model.decoder.layers.8.self_attn.k_proj.bias\n",
      "model.decoder.layers.8.self_attn.v_proj.weight\n",
      "model.decoder.layers.8.self_attn.v_proj.bias\n",
      "model.decoder.layers.8.self_attn.q_proj.weight\n",
      "model.decoder.layers.8.self_attn.q_proj.bias\n",
      "model.decoder.layers.8.self_attn.out_proj.weight\n",
      "model.decoder.layers.8.self_attn.out_proj.bias\n",
      "model.decoder.layers.8.self_attn_layer_norm.weight\n",
      "model.decoder.layers.8.self_attn_layer_norm.bias\n",
      "model.decoder.layers.8.encoder_attn.k_proj.weight\n",
      "model.decoder.layers.8.encoder_attn.k_proj.bias\n",
      "model.decoder.layers.8.encoder_attn.v_proj.weight\n",
      "model.decoder.layers.8.encoder_attn.v_proj.bias\n",
      "model.decoder.layers.8.encoder_attn.q_proj.weight\n",
      "model.decoder.layers.8.encoder_attn.q_proj.bias\n",
      "model.decoder.layers.8.encoder_attn.out_proj.weight\n",
      "model.decoder.layers.8.encoder_attn.out_proj.bias\n",
      "model.decoder.layers.8.encoder_attn_layer_norm.weight\n",
      "model.decoder.layers.8.encoder_attn_layer_norm.bias\n",
      "model.decoder.layers.8.fc1.weight\n",
      "model.decoder.layers.8.fc1.bias\n",
      "model.decoder.layers.8.fc2.weight\n",
      "model.decoder.layers.8.fc2.bias\n",
      "model.decoder.layers.8.final_layer_norm.weight\n",
      "model.decoder.layers.8.final_layer_norm.bias\n",
      "model.decoder.layers.9.self_attn.k_proj.weight\n",
      "model.decoder.layers.9.self_attn.k_proj.bias\n",
      "model.decoder.layers.9.self_attn.v_proj.weight\n",
      "model.decoder.layers.9.self_attn.v_proj.bias\n",
      "model.decoder.layers.9.self_attn.q_proj.weight\n",
      "model.decoder.layers.9.self_attn.q_proj.bias\n",
      "model.decoder.layers.9.self_attn.out_proj.weight\n",
      "model.decoder.layers.9.self_attn.out_proj.bias\n",
      "model.decoder.layers.9.self_attn_layer_norm.weight\n",
      "model.decoder.layers.9.self_attn_layer_norm.bias\n",
      "model.decoder.layers.9.encoder_attn.k_proj.weight\n",
      "model.decoder.layers.9.encoder_attn.k_proj.bias\n",
      "model.decoder.layers.9.encoder_attn.v_proj.weight\n",
      "model.decoder.layers.9.encoder_attn.v_proj.bias\n",
      "model.decoder.layers.9.encoder_attn.q_proj.weight\n",
      "model.decoder.layers.9.encoder_attn.q_proj.bias\n",
      "model.decoder.layers.9.encoder_attn.out_proj.weight\n",
      "model.decoder.layers.9.encoder_attn.out_proj.bias\n",
      "model.decoder.layers.9.encoder_attn_layer_norm.weight\n",
      "model.decoder.layers.9.encoder_attn_layer_norm.bias\n",
      "model.decoder.layers.9.fc1.weight\n",
      "model.decoder.layers.9.fc1.bias\n",
      "model.decoder.layers.9.fc2.weight\n",
      "model.decoder.layers.9.fc2.bias\n",
      "model.decoder.layers.9.final_layer_norm.weight\n",
      "model.decoder.layers.9.final_layer_norm.bias\n",
      "model.decoder.layers.10.self_attn.k_proj.weight\n",
      "model.decoder.layers.10.self_attn.k_proj.bias\n",
      "model.decoder.layers.10.self_attn.v_proj.weight\n",
      "model.decoder.layers.10.self_attn.v_proj.bias\n",
      "model.decoder.layers.10.self_attn.q_proj.weight\n",
      "model.decoder.layers.10.self_attn.q_proj.bias\n",
      "model.decoder.layers.10.self_attn.out_proj.weight\n",
      "model.decoder.layers.10.self_attn.out_proj.bias\n",
      "model.decoder.layers.10.self_attn_layer_norm.weight\n",
      "model.decoder.layers.10.self_attn_layer_norm.bias\n",
      "model.decoder.layers.10.encoder_attn.k_proj.weight\n",
      "model.decoder.layers.10.encoder_attn.k_proj.bias\n",
      "model.decoder.layers.10.encoder_attn.v_proj.weight\n",
      "model.decoder.layers.10.encoder_attn.v_proj.bias\n",
      "model.decoder.layers.10.encoder_attn.q_proj.weight\n",
      "model.decoder.layers.10.encoder_attn.q_proj.bias\n",
      "model.decoder.layers.10.encoder_attn.out_proj.weight\n",
      "model.decoder.layers.10.encoder_attn.out_proj.bias\n",
      "model.decoder.layers.10.encoder_attn_layer_norm.weight\n",
      "model.decoder.layers.10.encoder_attn_layer_norm.bias\n",
      "model.decoder.layers.10.fc1.weight\n",
      "model.decoder.layers.10.fc1.bias\n",
      "model.decoder.layers.10.fc2.weight\n",
      "model.decoder.layers.10.fc2.bias\n",
      "model.decoder.layers.10.final_layer_norm.weight\n",
      "model.decoder.layers.10.final_layer_norm.bias\n",
      "model.decoder.layers.11.self_attn.k_proj.weight\n",
      "model.decoder.layers.11.self_attn.k_proj.bias\n",
      "model.decoder.layers.11.self_attn.v_proj.weight\n",
      "model.decoder.layers.11.self_attn.v_proj.bias\n",
      "model.decoder.layers.11.self_attn.q_proj.weight\n",
      "model.decoder.layers.11.self_attn.q_proj.bias\n",
      "model.decoder.layers.11.self_attn.out_proj.weight\n",
      "model.decoder.layers.11.self_attn.out_proj.bias\n",
      "model.decoder.layers.11.self_attn_layer_norm.weight\n",
      "model.decoder.layers.11.self_attn_layer_norm.bias\n",
      "model.decoder.layers.11.encoder_attn.k_proj.weight\n",
      "model.decoder.layers.11.encoder_attn.k_proj.bias\n",
      "model.decoder.layers.11.encoder_attn.v_proj.weight\n",
      "model.decoder.layers.11.encoder_attn.v_proj.bias\n",
      "model.decoder.layers.11.encoder_attn.q_proj.weight\n",
      "model.decoder.layers.11.encoder_attn.q_proj.bias\n",
      "model.decoder.layers.11.encoder_attn.out_proj.weight\n",
      "model.decoder.layers.11.encoder_attn.out_proj.bias\n",
      "model.decoder.layers.11.encoder_attn_layer_norm.weight\n",
      "model.decoder.layers.11.encoder_attn_layer_norm.bias\n",
      "model.decoder.layers.11.fc1.weight\n",
      "model.decoder.layers.11.fc1.bias\n",
      "model.decoder.layers.11.fc2.weight\n",
      "model.decoder.layers.11.fc2.bias\n",
      "model.decoder.layers.11.final_layer_norm.weight\n",
      "model.decoder.layers.11.final_layer_norm.bias\n",
      "model.decoder.layer_norm.weight\n",
      "model.decoder.layer_norm.bias\n",
      "lm_head.weight\n"
     ]
    }
   ],
   "source": [
    "for name, weight in pruned_model.state_dict().items():\n",
    "    if torch.all(torch.eq(weight, model_statedict[name])):\n",
    "        print(name)\n",
    "    else:\n",
    "        print(name, \"DOESN'T MATCH\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f38aa923-a110-4f62-b6fd-4d14194a52d8",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tap",
   "language": "python",
   "name": "tap"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}