finetune_pruned_40percent_bloom.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0e7a688f-ae54-4190-8cdc-d7323de203f2",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/homebrew/lib/python3.10/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "import torch\n",
    "import pickle as pkl\n",
    "\n",
    "from transformers import AutoTokenizer\n",
    "from transformers import TrainingArguments, Trainer\n",
    "from transformers import DataCollatorForLanguageModeling\n",
    "\n",
    "from transformers.models.bloom.configuration_bloom import BloomConfig\n",
    "from pruning.pruned_bloom import PrunedBloomForCausalLM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f2244ad1-1d4e-4347-b81e-9ab0d95f7251",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load model and tokenizer\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"bigscience/bloom-560m\")\n",
    "context_length = 2048"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4b00fe18-86dd-47ec-9b19-152b0a8705f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load model\n",
    "weights_path = \"pruned_40percent_560m_bloom.pt\"\n",
    "state_dict_shapes_path = \"pruned_40percent_560m_bloom_state_dict_shapes.pkl\"\n",
    "\n",
    "bloom_config = BloomConfig(\n",
    "    vocab_size=250880,\n",
    "    hidden_size=1024,\n",
    "    n_layer=24,\n",
    "    n_head=16,\n",
    "    layer_norm_epsilon=1e-5,\n",
    "    initializer_range=0.02,\n",
    "    use_cache=True,\n",
    "    bos_token_id=1,\n",
    "    eos_token_id=2,\n",
    "    apply_residual_connection_post_layernorm=False,\n",
    "    hidden_dropout=0.0,\n",
    "    attention_dropout=0.0,\n",
    "    pretraining_tp=1,  # TP rank used when training with megatron\n",
    "    slow_but_exact=False,\n",
    "    attention_softmax_in_fp32=True,\n",
    "    bias_dropout_fusion=True,\n",
    "    masked_softmax_fusion=True,\n",
    "    offset_alibi=100,\n",
    "    pad_token_id=3,\n",
    "    seq_length=2048,\n",
    "    skip_bias_add=True,\n",
    "    skip_bias_add_qkv=False,\n",
    "    unk_token_id=0,\n",
    "    \n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "67e9595a-954a-4d65-bfb3-276bfe8f1020",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pruned_model = PrunedBloomForCausalLM(bloom_config, state_dict_shapes_path)\n",
    "pruned_model.load_state_dict(torch.load(weights_path))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f0210519-aef2-4cf9-ac53-ccf8055db421",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Load data\n",
    "split_percent = 0.95\n",
    "\n",
    "data = pkl.load(open(\"conv_dicts/530_human_filtered_conv_pairs.pkl\", \"rb\"))\n",
    "random.shuffle(data)\n",
    "split = int(split_percent * len(data))\n",
    "\n",
    "train_data = data[:split]\n",
    "val_data = data[split:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f9933752-56c5-491c-9576-af24c7cb9a65",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# tokenize data\n",
    "tokenizer.pad_token = tokenizer.eos_token\n",
    "\n",
    "def tokenize(data, tokenizer, context_length):\n",
    "    outputs = tokenizer(\n",
    "        data,\n",
    "        truncation=True,\n",
    "        max_length=context_length,\n",
    "        return_overflowing_tokens=True,\n",
    "        return_length=True,\n",
    "    )\n",
    "    \n",
    "    input_batch = []\n",
    "    for length, input_ids in zip(outputs[\"length\"], outputs[\"input_ids\"]):\n",
    "        if length < context_length:\n",
    "            input_batch.append(input_ids)\n",
    "            \n",
    "    return input_batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "a7c627c5-a30b-49fc-abad-e31315bce8a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "class DialogueDataset(torch.utils.data.Dataset):\n",
    "    def __init__(self, data_list, tokenizer, context_length):\n",
    "        self.data_strings = data_list\n",
    "        self.tokenized_data = tokenize(data_list, tokenizer, context_length)\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.data_strings)\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        return self.tokenized_data[idx].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "dc136740-3243-4612-8633-bde087dd8c1b",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = DialogueDataset(train_data, tokenizer, context_length)\n",
    "val_dataset = DialogueDataset(val_data, tokenizer, context_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "aa2c5918-51eb-4c8b-b7ea-175295f36b80",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "5545b6ec-cbad-4204-927a-f970a5cccf5e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Setup trainer args\n",
    "training_args = TrainingArguments(\n",
    "    output_dir=\"./results\",\n",
    "    evaluation_strategy=\"epoch\",\n",
    "    logging_strategy=\"steps\",\n",
    "    learning_rate=2e-5,\n",
    "    weight_decay=0.01,\n",
    "    num_train_epochs=3,\n",
    "    logging_steps=5,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "f754e571-7c23-4a88-abdf-98d8d9f4af41",
   "metadata": {},
   "outputs": [],
   "source": [
    "# init trainer\n",
    "trainer = Trainer(\n",
    "    model=pruned_model,\n",
    "    args=training_args,\n",
    "    train_dataset=train_dataset,\n",
    "    eval_dataset=val_dataset,\n",
    "    data_collator=data_collator,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "c506370d-def3-4f73-9420-5bca7e928708",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/homebrew/lib/python3.10/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n",
      "***** Running training *****\n",
      "  Num examples = 503\n",
      "  Num Epochs = 3\n",
      "  Instantaneous batch size per device = 8\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 8\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 189\n",
      "  Number of trainable parameters = 469695831\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='189' max='189' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [189/189 13:52, Epoch 3/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1.304000</td>\n",
       "      <td>1.073681</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.420400</td>\n",
       "      <td>0.430984</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.255500</td>\n",
       "      <td>0.303735</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "***** Running Evaluation *****\n",
      "  Num examples = 27\n",
      "  Batch size = 8\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 27\n",
      "  Batch size = 8\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 27\n",
      "  Batch size = 8\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=189, training_loss=0.9888538090640275, metrics={'train_runtime': 837.2838, 'train_samples_per_second': 1.802, 'train_steps_per_second': 0.226, 'total_flos': 241628394340500.0, 'train_loss': 0.9888538090640275, 'epoch': 3.0})"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "c0e21227-397a-45b1-8958-52403e4ae6b4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to ./results\n",
      "Configuration saved in ./results/config.json\n",
      "Configuration saved in ./results/generation_config.json\n",
      "Model weights saved in ./results/pytorch_model.bin\n"
     ]
    }
   ],
   "source": [
    "trainer.save_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "0a8d9afc-190a-4a38-b787-58b64e790e5f",
   "metadata": {},
   "outputs": [],
   "source": [
    "finetuned_model = trainer.model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "81610174-2070-4c38-8a1b-bfc8fb7b2c60",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Generate config GenerationConfig {\n",
      "  \"bos_token_id\": 1,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"pad_token_id\": 3,\n",
      "  \"transformers_version\": \"4.26.0\"\n",
      "}\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[\"user: Do you like watching movies?\\nchatbot: I absolutely love movies! Especially those with a lot of suspense and action. I love taking me to a different trip with my friends to watch some amazing movies. What type of movie do you usually like to watch? It's always a fun and interesting\"]\n"
     ]
    }
   ],
   "source": [
    "line = \"user: Do you like watching movies?\\nchatbot:\"\n",
    "inputs = tokenizer(line, return_tensors=\"pt\")\n",
    "pruned_times = []\n",
    "\n",
    "\n",
    "outputs = finetuned_model.generate(\n",
    "    input_ids=inputs[\"input_ids\"], \n",
    "    max_new_tokens=50, \n",
    "    do_sample=True, \n",
    "    top_k=50, \n",
    "    top_p=0.95,\n",
    ")\n",
    "\n",
    "print(tokenizer.batch_decode(outputs, skip_special_tokens=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "11b2dbe3-1cc0-468a-84ab-f81ab163f6b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(finetuned_model, \"finetuned_40percent_pruned_bloom560m.pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "f1f24181-4ce1-4a5a-902e-87076f9c7c06",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Generate config GenerationConfig {\n",
      "  \"bos_token_id\": 1,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"pad_token_id\": 3,\n",
      "  \"transformers_version\": \"4.26.0\"\n",
      "}\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[\"user: Can you tell me about yourself?\\nchatbot: Yes, of course! Anime is one of my favorite genres of work to be able to talk about. I love stories with a lot of suspense and action, and I'm always excited to learn more about the characters and the underlying storylines.\"]\n"
     ]
    }
   ],
   "source": [
    "line = \"user: Can you tell me about yourself?\\nchatbot:\"\n",
    "inputs = tokenizer(line, return_tensors=\"pt\")\n",
    "pruned_times = []\n",
    "\n",
    "\n",
    "outputs = finetuned_model.generate(\n",
    "    input_ids=inputs[\"input_ids\"], \n",
    "    max_new_tokens=50, \n",
    "    do_sample=True, \n",
    "    top_k=50, \n",
    "    top_p=0.95,\n",
    ")\n",
    "\n",
    "print(tokenizer.batch_decode(outputs, skip_special_tokens=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "0fcd74af-4f64-4a23-944e-a7511c90aed6",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Generate config GenerationConfig {\n",
      "  \"bos_token_id\": 1,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"pad_token_id\": 3,\n",
      "  \"transformers_version\": \"4.26.0\"\n",
      "}\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[\"user: what do you want to talk about?\\nchatbot: I want to talk about the power of technology and the potential to make a real difference in the world. I'm here to help\"]\n"
     ]
    }
   ],
   "source": [
    "line = \"user: what do you want to talk about?\\nchatbot:\"\n",
    "inputs = tokenizer(line, return_tensors=\"pt\")\n",
    "pruned_times = []\n",
    "\n",
    "\n",
    "outputs = finetuned_model.generate(\n",
    "    input_ids=inputs[\"input_ids\"], \n",
    "    max_new_tokens=25, \n",
    "    do_sample=True, \n",
    "    top_k=50, \n",
    "    top_p=0.95,\n",
    ")\n",
    "\n",
    "print(tokenizer.batch_decode(outputs, skip_special_tokens=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "61118108-0d45-4176-b2ce-675dfe500523",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Generate config GenerationConfig {\n",
      "  \"bos_token_id\": 1,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"pad_token_id\": 3,\n",
      "  \"transformers_version\": \"4.26.0\"\n",
      "}\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[\"user: you misunderstood me\\nchatbot: That is totally true. I understand that you love Alpha and Omega: Dino Digs and I am passionate about making sure that everyone's voice is heard in the future. I'm here to help learn more about how to be an effective advocate for\"]\n"
     ]
    }
   ],
   "source": [
    "line = \"user: you misunderstood me\\nchatbot:\"\n",
    "inputs = tokenizer(line, return_tensors=\"pt\")\n",
    "pruned_times = []\n",
    "\n",
    "\n",
    "outputs = finetuned_model.generate(\n",
    "    input_ids=inputs[\"input_ids\"], \n",
    "    max_new_tokens=50, \n",
    "    do_sample=True, \n",
    "    top_k=50, \n",
    "    top_p=0.95,\n",
    ")\n",
    "\n",
    "print(tokenizer.batch_decode(outputs, skip_special_tokens=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "2c0ef884-f79e-403c-9bd8-eec27f682ef4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      " Hi, what do you want to talk about?\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Generate config GenerationConfig {\n",
      "  \"bos_token_id\": 1,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"pad_token_id\": 3,\n",
      "  \"transformers_version\": \"4.26.0\"\n",
      "}\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Hi, what are some of your favorite movies? I'm always interested to hear what other people think about the movies they are watching\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      " my favorite movie is inception\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Generate config GenerationConfig {\n",
      "  \"bos_token_id\": 1,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"pad_token_id\": 3,\n",
      "  \"transformers_version\": \"4.26.0\"\n",
      "}\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Absolutely! It's a great movie with a great message. Have you seen any other movies that you liked? What genre\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      " I really like movies in the thiller genre, what about you?\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Generate config GenerationConfig {\n",
      "  \"bos_token_id\": 1,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"pad_token_id\": 3,\n",
      "  \"transformers_version\": \"4.26.0\"\n",
      "}\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  I completely agree with you! The thriller novel is an intricate plotline and a great reminder of how far we\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800000; text-decoration-color: #800000\">╭─────────────────────────────── </span><span style=\"color: #800000; text-decoration-color: #800000; font-weight: bold\">Traceback </span><span style=\"color: #bf7f7f; text-decoration-color: #bf7f7f; font-weight: bold\">(most recent call last)</span><span style=\"color: #800000; text-decoration-color: #800000\"> ────────────────────────────────╮</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #bfbf7f; text-decoration-color: #bfbf7f\">/var/folders/qm/qmg6cwns11x93fg6_77cntd40000gn/T/ipykernel_60240/</span><span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">3081165780.py</span>:<span style=\"color: #0000ff; text-decoration-color: #0000ff\">2</span> in <span style=\"color: #00ff00; text-decoration-color: #00ff00\">&lt;module&gt;</span>     <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>                                                                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000; font-style: italic\">[Errno 2] No such file or directory: </span>                                                            <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000; font-style: italic\">'/var/folders/qm/qmg6cwns11x93fg6_77cntd40000gn/T/ipykernel_60240/3081165780.py'</span>                 <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>                                                                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #bfbf7f; text-decoration-color: #bfbf7f\">/opt/homebrew/lib/python3.10/site-packages/ipykernel/</span><span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">kernelbase.py</span>:<span style=\"color: #0000ff; text-decoration-color: #0000ff\">1182</span> in <span style=\"color: #00ff00; text-decoration-color: #00ff00\">raw_input</span>             <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>                                                                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1179 │   │   </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">if</span> <span style=\"color: #ff00ff; text-decoration-color: #ff00ff\">not</span> <span style=\"color: #00ffff; text-decoration-color: #00ffff\">self</span>._allow_stdin:                                                         <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1180 │   │   │   </span>msg = <span style=\"color: #808000; text-decoration-color: #808000\">\"raw_input was called, but this frontend does not support input reques</span>  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1181 │   │   │   </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">raise</span> StdinNotImplementedError(msg)                                           <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">❱ </span>1182 <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">return</span> <span style=\"color: #00ffff; text-decoration-color: #00ffff\">self</span>._input_request(                                                       <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1183 │   │   │   </span><span style=\"color: #00ffff; text-decoration-color: #00ffff\">str</span>(prompt),                                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1184 │   │   │   </span><span style=\"color: #00ffff; text-decoration-color: #00ffff\">self</span>._parent_ident[<span style=\"color: #808000; text-decoration-color: #808000\">\"shell\"</span>],                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1185 │   │   │   </span><span style=\"color: #00ffff; text-decoration-color: #00ffff\">self</span>.get_parent(<span style=\"color: #808000; text-decoration-color: #808000\">\"shell\"</span>),                                                     <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>                                                                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #bfbf7f; text-decoration-color: #bfbf7f\">/opt/homebrew/lib/python3.10/site-packages/ipykernel/</span><span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">kernelbase.py</span>:<span style=\"color: #0000ff; text-decoration-color: #0000ff\">1225</span> in <span style=\"color: #00ff00; text-decoration-color: #00ff00\">_input_request</span>        <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>                                                                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1222 │   │   │   </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">except</span> <span style=\"color: #00ffff; text-decoration-color: #00ffff\">KeyboardInterrupt</span>:                                                     <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1223 │   │   │   │   # re-raise KeyboardInterrupt, to truncate traceback</span>                       <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1224 │   │   │   │   </span>msg = <span style=\"color: #808000; text-decoration-color: #808000\">\"Interrupted by user\"</span>                                               <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">❱ </span>1225 <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   │   │   </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">raise</span> <span style=\"color: #00ffff; text-decoration-color: #00ffff\">KeyboardInterrupt</span>(msg) <span style=\"color: #0000ff; text-decoration-color: #0000ff\">from</span> <span style=\"color: #00ffff; text-decoration-color: #00ffff\">None</span>                                    <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1226 │   │   │   </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">except</span> <span style=\"color: #00ffff; text-decoration-color: #00ffff\">Exception</span>:                                                             <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1227 │   │   │   │   </span><span style=\"color: #00ffff; text-decoration-color: #00ffff\">self</span>.log.warning(<span style=\"color: #808000; text-decoration-color: #808000\">\"Invalid Message:\"</span>, exc_info=<span style=\"color: #0000ff; text-decoration-color: #0000ff\">True</span>)                       <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1228 </span>                                                                                          <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">╰──────────────────────────────────────────────────────────────────────────────────────────────────╯</span>\n",
       "<span style=\"color: #ff0000; text-decoration-color: #ff0000; font-weight: bold\">KeyboardInterrupt: </span>Interrupted by user\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n",
       "\u001b[31m│\u001b[0m \u001b[2;33m/var/folders/qm/qmg6cwns11x93fg6_77cntd40000gn/T/ipykernel_60240/\u001b[0m\u001b[1;33m3081165780.py\u001b[0m:\u001b[94m2\u001b[0m in \u001b[92m<module>\u001b[0m     \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m                                                                                                  \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m \u001b[3;31m[Errno 2] No such file or directory: \u001b[0m                                                            \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m \u001b[3;31m'/var/folders/qm/qmg6cwns11x93fg6_77cntd40000gn/T/ipykernel_60240/3081165780.py'\u001b[0m                 \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m                                                                                                  \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m \u001b[2;33m/opt/homebrew/lib/python3.10/site-packages/ipykernel/\u001b[0m\u001b[1;33mkernelbase.py\u001b[0m:\u001b[94m1182\u001b[0m in \u001b[92mraw_input\u001b[0m             \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m                                                                                                  \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m   \u001b[2m1179 \u001b[0m\u001b[2m│   │   \u001b[0m\u001b[94mif\u001b[0m \u001b[95mnot\u001b[0m \u001b[96mself\u001b[0m._allow_stdin:                                                         \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m   \u001b[2m1180 \u001b[0m\u001b[2m│   │   │   \u001b[0mmsg = \u001b[33m\"\u001b[0m\u001b[33mraw_input was called, but this frontend does not support input reques\u001b[0m  \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m   \u001b[2m1181 \u001b[0m\u001b[2m│   │   │   \u001b[0m\u001b[94mraise\u001b[0m StdinNotImplementedError(msg)                                           \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1182 \u001b[2m│   │   \u001b[0m\u001b[94mreturn\u001b[0m \u001b[96mself\u001b[0m._input_request(                                                       \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m   \u001b[2m1183 \u001b[0m\u001b[2m│   │   │   \u001b[0m\u001b[96mstr\u001b[0m(prompt),                                                                  \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m   \u001b[2m1184 \u001b[0m\u001b[2m│   │   │   \u001b[0m\u001b[96mself\u001b[0m._parent_ident[\u001b[33m\"\u001b[0m\u001b[33mshell\u001b[0m\u001b[33m\"\u001b[0m],                                                  \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m   \u001b[2m1185 \u001b[0m\u001b[2m│   │   │   \u001b[0m\u001b[96mself\u001b[0m.get_parent(\u001b[33m\"\u001b[0m\u001b[33mshell\u001b[0m\u001b[33m\"\u001b[0m),                                                     \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m                                                                                                  \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m \u001b[2;33m/opt/homebrew/lib/python3.10/site-packages/ipykernel/\u001b[0m\u001b[1;33mkernelbase.py\u001b[0m:\u001b[94m1225\u001b[0m in \u001b[92m_input_request\u001b[0m        \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m                                                                                                  \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m   \u001b[2m1222 \u001b[0m\u001b[2m│   │   │   \u001b[0m\u001b[94mexcept\u001b[0m \u001b[96mKeyboardInterrupt\u001b[0m:                                                     \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m   \u001b[2m1223 \u001b[0m\u001b[2m│   │   │   │   \u001b[0m\u001b[2m# re-raise KeyboardInterrupt, to truncate traceback\u001b[0m                       \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m   \u001b[2m1224 \u001b[0m\u001b[2m│   │   │   │   \u001b[0mmsg = \u001b[33m\"\u001b[0m\u001b[33mInterrupted by user\u001b[0m\u001b[33m\"\u001b[0m                                               \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1225 \u001b[2m│   │   │   │   \u001b[0m\u001b[94mraise\u001b[0m \u001b[96mKeyboardInterrupt\u001b[0m(msg) \u001b[94mfrom\u001b[0m \u001b[96mNone\u001b[0m                                    \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m   \u001b[2m1226 \u001b[0m\u001b[2m│   │   │   \u001b[0m\u001b[94mexcept\u001b[0m \u001b[96mException\u001b[0m:                                                             \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m   \u001b[2m1227 \u001b[0m\u001b[2m│   │   │   │   \u001b[0m\u001b[96mself\u001b[0m.log.warning(\u001b[33m\"\u001b[0m\u001b[33mInvalid Message:\u001b[0m\u001b[33m\"\u001b[0m, exc_info=\u001b[94mTrue\u001b[0m)                       \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m   \u001b[2m1228 \u001b[0m                                                                                          \u001b[31m│\u001b[0m\n",
       "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
       "\u001b[1;91mKeyboardInterrupt: \u001b[0mInterrupted by user\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "while True:\n",
    "    user_input = input().strip()\n",
    "    line = f\"user: {user_input}\\nchatbot:\"\n",
    "    inputs = tokenizer(line, return_tensors=\"pt\")\n",
    "    pruned_times = []\n",
    "\n",
    "\n",
    "    outputs = finetuned_model.generate(\n",
    "        input_ids=inputs[\"input_ids\"], \n",
    "        max_new_tokens=25, \n",
    "        do_sample=True, \n",
    "        top_k=50, \n",
    "        top_p=0.95,\n",
    "    )\n",
    "    \n",
    "\n",
    "    textoutput = tokenizer.batch_decode(outputs, skip_special_tokens=True)[0]\n",
    "    bot_response = textoutput.split(\"\\nchatbot:\")[1]\n",
    "    print(bot_response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "41f22781-6a32-4083-828e-2fe843009abd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      " Hi, what do you want to talk about?\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Generate config GenerationConfig {\n",
      "  \"bos_token_id\": 1,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"pad_token_id\": 3,\n",
      "  \"transformers_version\": \"4.26.0\"\n",
      "}\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " I think exploring the great outdoors is my favorite activity by far. I'm particularly fond of hiking, exploring the nature,\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      " I also like exploring nature, have you recently been camping?\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Generate config GenerationConfig {\n",
      "  \"bos_token_id\": 1,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"pad_token_id\": 3,\n",
      "  \"transformers_version\": \"4.26.0\"\n",
      "}\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Yes, I've been camping a bit lately. The most rewarding part was getting back into the real world. The scall\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      " how long was your camping trip?\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Generate config GenerationConfig {\n",
      "  \"bos_token_id\": 1,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"pad_token_id\": 3,\n",
      "  \"transformers_version\": \"4.26.0\"\n",
      "}\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Sure thing! My camping trip was a combination of adventure and peace. I took two-hour-long and fourchatbot's\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      " i didn't understand that last bit\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Generate config GenerationConfig {\n",
      "  \"bos_token_id\": 1,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"pad_token_id\": 3,\n",
      "  \"transformers_version\": \"4.26.0\"\n",
      "}\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " That sounds like a fascinating story! It's incredible how Young Justice is able to bring the story of DC comics to life\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800000; text-decoration-color: #800000\">╭─────────────────────────────── </span><span style=\"color: #800000; text-decoration-color: #800000; font-weight: bold\">Traceback </span><span style=\"color: #bf7f7f; text-decoration-color: #bf7f7f; font-weight: bold\">(most recent call last)</span><span style=\"color: #800000; text-decoration-color: #800000\"> ────────────────────────────────╮</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #bfbf7f; text-decoration-color: #bfbf7f\">/var/folders/qm/qmg6cwns11x93fg6_77cntd40000gn/T/ipykernel_60240/</span><span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">352736422.py</span>:<span style=\"color: #0000ff; text-decoration-color: #0000ff\">3</span> in <span style=\"color: #00ff00; text-decoration-color: #00ff00\">&lt;module&gt;</span>      <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>                                                                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000; font-style: italic\">[Errno 2] No such file or directory: </span>                                                            <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000; font-style: italic\">'/var/folders/qm/qmg6cwns11x93fg6_77cntd40000gn/T/ipykernel_60240/352736422.py'</span>                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>                                                                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #bfbf7f; text-decoration-color: #bfbf7f\">/opt/homebrew/lib/python3.10/site-packages/ipykernel/</span><span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">kernelbase.py</span>:<span style=\"color: #0000ff; text-decoration-color: #0000ff\">1182</span> in <span style=\"color: #00ff00; text-decoration-color: #00ff00\">raw_input</span>             <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>                                                                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1179 │   │   </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">if</span> <span style=\"color: #ff00ff; text-decoration-color: #ff00ff\">not</span> <span style=\"color: #00ffff; text-decoration-color: #00ffff\">self</span>._allow_stdin:                                                         <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1180 │   │   │   </span>msg = <span style=\"color: #808000; text-decoration-color: #808000\">\"raw_input was called, but this frontend does not support input reques</span>  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1181 │   │   │   </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">raise</span> StdinNotImplementedError(msg)                                           <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">❱ </span>1182 <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">return</span> <span style=\"color: #00ffff; text-decoration-color: #00ffff\">self</span>._input_request(                                                       <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1183 │   │   │   </span><span style=\"color: #00ffff; text-decoration-color: #00ffff\">str</span>(prompt),                                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1184 │   │   │   </span><span style=\"color: #00ffff; text-decoration-color: #00ffff\">self</span>._parent_ident[<span style=\"color: #808000; text-decoration-color: #808000\">\"shell\"</span>],                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1185 │   │   │   </span><span style=\"color: #00ffff; text-decoration-color: #00ffff\">self</span>.get_parent(<span style=\"color: #808000; text-decoration-color: #808000\">\"shell\"</span>),                                                     <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>                                                                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #bfbf7f; text-decoration-color: #bfbf7f\">/opt/homebrew/lib/python3.10/site-packages/ipykernel/</span><span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">kernelbase.py</span>:<span style=\"color: #0000ff; text-decoration-color: #0000ff\">1225</span> in <span style=\"color: #00ff00; text-decoration-color: #00ff00\">_input_request</span>        <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>                                                                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1222 │   │   │   </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">except</span> <span style=\"color: #00ffff; text-decoration-color: #00ffff\">KeyboardInterrupt</span>:                                                     <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1223 │   │   │   │   # re-raise KeyboardInterrupt, to truncate traceback</span>                       <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1224 │   │   │   │   </span>msg = <span style=\"color: #808000; text-decoration-color: #808000\">\"Interrupted by user\"</span>                                               <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">❱ </span>1225 <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   │   │   </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">raise</span> <span style=\"color: #00ffff; text-decoration-color: #00ffff\">KeyboardInterrupt</span>(msg) <span style=\"color: #0000ff; text-decoration-color: #0000ff\">from</span> <span style=\"color: #00ffff; text-decoration-color: #00ffff\">None</span>                                    <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1226 │   │   │   </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">except</span> <span style=\"color: #00ffff; text-decoration-color: #00ffff\">Exception</span>:                                                             <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1227 │   │   │   │   </span><span style=\"color: #00ffff; text-decoration-color: #00ffff\">self</span>.log.warning(<span style=\"color: #808000; text-decoration-color: #808000\">\"Invalid Message:\"</span>, exc_info=<span style=\"color: #0000ff; text-decoration-color: #0000ff\">True</span>)                       <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1228 </span>                                                                                          <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">╰──────────────────────────────────────────────────────────────────────────────────────────────────╯</span>\n",
       "<span style=\"color: #ff0000; text-decoration-color: #ff0000; font-weight: bold\">KeyboardInterrupt: </span>Interrupted by user\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n",
       "\u001b[31m│\u001b[0m \u001b[2;33m/var/folders/qm/qmg6cwns11x93fg6_77cntd40000gn/T/ipykernel_60240/\u001b[0m\u001b[1;33m352736422.py\u001b[0m:\u001b[94m3\u001b[0m in \u001b[92m<module>\u001b[0m      \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m                                                                                                  \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m \u001b[3;31m[Errno 2] No such file or directory: \u001b[0m                                                            \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m \u001b[3;31m'/var/folders/qm/qmg6cwns11x93fg6_77cntd40000gn/T/ipykernel_60240/352736422.py'\u001b[0m                  \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m                                                                                                  \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m \u001b[2;33m/opt/homebrew/lib/python3.10/site-packages/ipykernel/\u001b[0m\u001b[1;33mkernelbase.py\u001b[0m:\u001b[94m1182\u001b[0m in \u001b[92mraw_input\u001b[0m             \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m                                                                                                  \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m   \u001b[2m1179 \u001b[0m\u001b[2m│   │   \u001b[0m\u001b[94mif\u001b[0m \u001b[95mnot\u001b[0m \u001b[96mself\u001b[0m._allow_stdin:                                                         \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m   \u001b[2m1180 \u001b[0m\u001b[2m│   │   │   \u001b[0mmsg = \u001b[33m\"\u001b[0m\u001b[33mraw_input was called, but this frontend does not support input reques\u001b[0m  \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m   \u001b[2m1181 \u001b[0m\u001b[2m│   │   │   \u001b[0m\u001b[94mraise\u001b[0m StdinNotImplementedError(msg)                                           \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1182 \u001b[2m│   │   \u001b[0m\u001b[94mreturn\u001b[0m \u001b[96mself\u001b[0m._input_request(                                                       \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m   \u001b[2m1183 \u001b[0m\u001b[2m│   │   │   \u001b[0m\u001b[96mstr\u001b[0m(prompt),                                                                  \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m   \u001b[2m1184 \u001b[0m\u001b[2m│   │   │   \u001b[0m\u001b[96mself\u001b[0m._parent_ident[\u001b[33m\"\u001b[0m\u001b[33mshell\u001b[0m\u001b[33m\"\u001b[0m],                                                  \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m   \u001b[2m1185 \u001b[0m\u001b[2m│   │   │   \u001b[0m\u001b[96mself\u001b[0m.get_parent(\u001b[33m\"\u001b[0m\u001b[33mshell\u001b[0m\u001b[33m\"\u001b[0m),                                                     \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m                                                                                                  \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m \u001b[2;33m/opt/homebrew/lib/python3.10/site-packages/ipykernel/\u001b[0m\u001b[1;33mkernelbase.py\u001b[0m:\u001b[94m1225\u001b[0m in \u001b[92m_input_request\u001b[0m        \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m                                                                                                  \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m   \u001b[2m1222 \u001b[0m\u001b[2m│   │   │   \u001b[0m\u001b[94mexcept\u001b[0m \u001b[96mKeyboardInterrupt\u001b[0m:                                                     \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m   \u001b[2m1223 \u001b[0m\u001b[2m│   │   │   │   \u001b[0m\u001b[2m# re-raise KeyboardInterrupt, to truncate traceback\u001b[0m                       \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m   \u001b[2m1224 \u001b[0m\u001b[2m│   │   │   │   \u001b[0mmsg = \u001b[33m\"\u001b[0m\u001b[33mInterrupted by user\u001b[0m\u001b[33m\"\u001b[0m                                               \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1225 \u001b[2m│   │   │   │   \u001b[0m\u001b[94mraise\u001b[0m \u001b[96mKeyboardInterrupt\u001b[0m(msg) \u001b[94mfrom\u001b[0m \u001b[96mNone\u001b[0m                                    \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m   \u001b[2m1226 \u001b[0m\u001b[2m│   │   │   \u001b[0m\u001b[94mexcept\u001b[0m \u001b[96mException\u001b[0m:                                                             \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m   \u001b[2m1227 \u001b[0m\u001b[2m│   │   │   │   \u001b[0m\u001b[96mself\u001b[0m.log.warning(\u001b[33m\"\u001b[0m\u001b[33mInvalid Message:\u001b[0m\u001b[33m\"\u001b[0m, exc_info=\u001b[94mTrue\u001b[0m)                       \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m   \u001b[2m1228 \u001b[0m                                                                                          \u001b[31m│\u001b[0m\n",
       "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
       "\u001b[1;91mKeyboardInterrupt: \u001b[0mInterrupted by user\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "while True:\n",
    "    chat_history = \"\"\n",
    "    user_input = input().strip()\n",
    "    line = f\"user: {user_input}\\nchatbot:\"\n",
    "    chat_history += line\n",
    "    inputs = tokenizer(chat_history, return_tensors=\"pt\")\n",
    "\n",
    "\n",
    "    outputs = finetuned_model.generate(\n",
    "        input_ids=inputs[\"input_ids\"], \n",
    "        max_new_tokens=25, \n",
    "        do_sample=True, \n",
    "        top_k=50, \n",
    "        top_p=0.95,\n",
    "    )\n",
    "    \n",
    "\n",
    "    textoutput = tokenizer.batch_decode(outputs, skip_special_tokens=True)[0]\n",
    "    bot_response = textoutput.split(\"\\nchatbot:\")[1]\n",
    "    print(bot_response)\n",
    "    chat_history += bot_response + \"\\n\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c518bfce-c5af-497e-b546-37b1d72f18db",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tap",
   "language": "python",
   "name": "tap"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}