examples/llm_eval.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ[\"CLARIFAI_PAT\"] = \"pat\"\n",
    "\n",
    "from clarifai_model_utils import ClarifaiEvaluator\n",
    "from clarifai_model_utils.llm_eval.constant import JUDGE_LLMS\n",
    "\n",
    "from clarifai.client.model import Model\n",
    "from clarifai.client.dataset import Dataset\n",
    "\n",
    "model = Model(\n",
    "    url=\"https://clarifai.com/phatvo/lm_eval/models/dummy_text/model_version_id/c630093c80104d2582a2b76132efd64a\")\n",
    "\n",
    "ds = Dataset(url=\"https://clarifai.com/phatvo/lm_eval/datasets/alpaca-eval-5\")\n",
    "\n",
    "evaluator = ClarifaiEvaluator(predictor=model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# evaluate with above dataset using `general` template and not upload eval result to cloud\n",
    "out = evaluator.evaluate(\n",
    "    template=\"general\",\n",
    "    upload=False,\n",
    "    dataset=ds,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "id='general,lm_eval,alpaca-eval-5,1709024737' template='general' dataset=DatasetInfo(id='alpaca-eval-5', app_id='lm_eval', user_id='', version_id='') df={'question': ['What is some cool music from the 1920s?', 'What are the names of some famous actors that started their careers on Broadway?', \"Hi, my sister and her girlfriends want me to play kickball with them. Can you explain how the game is played, so they don't take advantage of me?\", 'How do I wrap a present neatly?', 'How did US states get their names?'], 'answer': ['Some cool music from the 1920s includes jazz classics like Duke Ellington\\'s \"Mood Indigo,\" Louis Armstrong\\'s \"West End Blues,\" and Bessie Smith\\'s \"Down Hearted Blues.\" Other popular tunes from the era include Fats Waller\\'s \"Ain\\'t Misbehavin,\" George Gershwin\\'s \"Rhapsody in Blue,\" and Irving Berlin\\'s \"Puttin\\' On the Ritz.\"', 'Some famous actors that started their careers on Broadway include: \\n1. Hugh Jackman \\n2. Meryl Streep \\n3. Denzel Washington \\n4. Julia Roberts \\n5. Christopher Walken \\n6. Anthony Rapp \\n7. Audra McDonald \\n8. Nathan Lane \\n9. Sarah Jessica Parker \\n10. Lin-Manuel Miranda', 'Kickball is a game similar to baseball, but with a large rubber ball instead of a bat and a ball. The game is usually played with two teams of six players each. Each team has three bases and a home plate. The players on the kicking team line up at home plate and take turns kicking the ball. The object of the game is to score runs by running around all three bases and back to home plate without being tagged out by the defense. The team with the most runs at the end of the game is the winner.', '1. Start by gathering the supplies you will need to wrap the present, such as wrapping paper, tape, scissors, and ribbon.\\n\\n2. Cut the wrapping paper to the size of the gift, ensuring that there is enough paper to cover the entire gift.\\n\\n3. Place the gift in the center of the wrapping paper, and fold the sides in, tucking the excess paper at the bottom.\\n\\n4. Secure the folded paper with tape.\\n\\n5. Cut a length of ribbon and tie it around the gift.\\n\\n6. Trim the excess ribbon with scissors.\\n\\n7. Add a decorative bow or other embellishment to the top of the gift.', 'US states get their names from a variety of sources, including Native American tribes, Spanish explorers, British colonists, and even presidents. For example, the state of Alabama was named after the Native American tribe that lived in the area, while the state of Florida gets its name from the Spanish explorer, Ponce de Leon, who explored the area in the 1500s. Other states are named after English kings (like Virginia, named after England\\'s \"Virgin Queen,\" Queen Elizabeth I) or presidents (like Washington, named after George Washington).'], 'prediction': ['Dummy output: What is some cool music from the 1920s?', 'Dummy output: What are the names of some famous actors that started their careers on Broadway?', \"Dummy output: Hi, my sister and her girlfriends want me to play kickball with them. Can you explain how the game is played, so they don't take advantage of me?\", 'Dummy output: How do I wrap a present neatly?', 'Dummy output: How did US states get their names?'], 'filtered_prediction': ['Dummy output: What is some cool music from the 1920s?', 'Dummy output: What are the names of some famous actors that started their careers on Broadway?', \"Dummy output: Hi, my sister and her girlfriends want me to play kickball with them. Can you explain how the game is played, so they don't take advantage of me?\", 'Dummy output: How do I wrap a present neatly?', 'Dummy output: How did US states get their names?'], 'exact_match': [0.0, 0.0, 0.0, 0.0, 0.0], 'f1': [18.518518518518515, 32.142857142857146, 16.071428571428573, 4.210526315789474, 11.494252873563218], 'sacrebleu': [0.10090157153750948, 5.420549568307133, 0.7739219525266074, 0.00012978404523164151, 0.00519309553348583], 'rouge1': [18.46153846153846, 31.034482758620683, 15.267175572519081, 5.217391304347826, 10.416666666666666], 'rouge2': [15.873015873015875, 28.57142857142857, 3.10077519379845, 0.0, 8.51063829787234], 'rougeL': [18.46153846153846, 31.034482758620683, 12.213740458015266, 3.4782608695652173, 10.416666666666666]} summary={'exact_match': 0.0, 'f1': 16.487516684431384, 'sacrebleu': 1.2601391943899936, 'rouge1': 16.079450952738547, 'rouge2': 11.211171587223046, 'rougeL': 15.120937842881258} weights={} llm_judge_info=None inference_params={} prompter=None regex_code='' timestamp=1709024737 meta={} average=10.026536043610704\n",
      "{'id': 'general,lm_eval,alpaca-eval-5,1709024737', 'template': 'general', 'dataset': {'id': 'alpaca-eval-5', 'app_id': 'lm_eval', 'user_id': '', 'version_id': ''}, 'df': {'question': ['What is some cool music from the 1920s?', 'What are the names of some famous actors that started their careers on Broadway?', \"Hi, my sister and her girlfriends want me to play kickball with them. Can you explain how the game is played, so they don't take advantage of me?\", 'How do I wrap a present neatly?', 'How did US states get their names?'], 'answer': ['Some cool music from the 1920s includes jazz classics like Duke Ellington\\'s \"Mood Indigo,\" Louis Armstrong\\'s \"West End Blues,\" and Bessie Smith\\'s \"Down Hearted Blues.\" Other popular tunes from the era include Fats Waller\\'s \"Ain\\'t Misbehavin,\" George Gershwin\\'s \"Rhapsody in Blue,\" and Irving Berlin\\'s \"Puttin\\' On the Ritz.\"', 'Some famous actors that started their careers on Broadway include: \\n1. Hugh Jackman \\n2. Meryl Streep \\n3. Denzel Washington \\n4. Julia Roberts \\n5. Christopher Walken \\n6. Anthony Rapp \\n7. Audra McDonald \\n8. Nathan Lane \\n9. Sarah Jessica Parker \\n10. Lin-Manuel Miranda', 'Kickball is a game similar to baseball, but with a large rubber ball instead of a bat and a ball. The game is usually played with two teams of six players each. Each team has three bases and a home plate. The players on the kicking team line up at home plate and take turns kicking the ball. The object of the game is to score runs by running around all three bases and back to home plate without being tagged out by the defense. The team with the most runs at the end of the game is the winner.', '1. Start by gathering the supplies you will need to wrap the present, such as wrapping paper, tape, scissors, and ribbon.\\n\\n2. Cut the wrapping paper to the size of the gift, ensuring that there is enough paper to cover the entire gift.\\n\\n3. Place the gift in the center of the wrapping paper, and fold the sides in, tucking the excess paper at the bottom.\\n\\n4. Secure the folded paper with tape.\\n\\n5. Cut a length of ribbon and tie it around the gift.\\n\\n6. Trim the excess ribbon with scissors.\\n\\n7. Add a decorative bow or other embellishment to the top of the gift.', 'US states get their names from a variety of sources, including Native American tribes, Spanish explorers, British colonists, and even presidents. For example, the state of Alabama was named after the Native American tribe that lived in the area, while the state of Florida gets its name from the Spanish explorer, Ponce de Leon, who explored the area in the 1500s. Other states are named after English kings (like Virginia, named after England\\'s \"Virgin Queen,\" Queen Elizabeth I) or presidents (like Washington, named after George Washington).'], 'prediction': ['Dummy output: What is some cool music from the 1920s?', 'Dummy output: What are the names of some famous actors that started their careers on Broadway?', \"Dummy output: Hi, my sister and her girlfriends want me to play kickball with them. Can you explain how the game is played, so they don't take advantage of me?\", 'Dummy output: How do I wrap a present neatly?', 'Dummy output: How did US states get their names?'], 'filtered_prediction': ['Dummy output: What is some cool music from the 1920s?', 'Dummy output: What are the names of some famous actors that started their careers on Broadway?', \"Dummy output: Hi, my sister and her girlfriends want me to play kickball with them. Can you explain how the game is played, so they don't take advantage of me?\", 'Dummy output: How do I wrap a present neatly?', 'Dummy output: How did US states get their names?'], 'exact_match': [0.0, 0.0, 0.0, 0.0, 0.0], 'f1': [18.518518518518515, 32.142857142857146, 16.071428571428573, 4.210526315789474, 11.494252873563218], 'sacrebleu': [0.10090157153750948, 5.420549568307133, 0.7739219525266074, 0.00012978404523164151, 0.00519309553348583], 'rouge1': [18.46153846153846, 31.034482758620683, 15.267175572519081, 5.217391304347826, 10.416666666666666], 'rouge2': [15.873015873015875, 28.57142857142857, 3.10077519379845, 0.0, 8.51063829787234], 'rougeL': [18.46153846153846, 31.034482758620683, 12.213740458015266, 3.4782608695652173, 10.416666666666666]}, 'summary': {'exact_match': 0.0, 'f1': 16.487516684431384, 'sacrebleu': 1.2601391943899936, 'rouge1': 16.079450952738547, 'rouge2': 11.211171587223046, 'rougeL': 15.120937842881258}, 'weights': {}, 'llm_judge_info': None, 'inference_params': {}, 'prompter': None, 'regex_code': '', 'meta': {}, 'average': 10.026536043610704}\n"
     ]
    }
   ],
   "source": [
    "print(out)\n",
    "print(out.model_dump())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question</th>\n",
       "      <th>answer</th>\n",
       "      <th>prediction</th>\n",
       "      <th>filtered_prediction</th>\n",
       "      <th>exact_match</th>\n",
       "      <th>f1</th>\n",
       "      <th>sacrebleu</th>\n",
       "      <th>rouge1</th>\n",
       "      <th>rouge2</th>\n",
       "      <th>rougeL</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>What is some cool music from the 1920s?</td>\n",
       "      <td>Some cool music from the 1920s includes jazz c...</td>\n",
       "      <td>Dummy output: What is some cool music from the...</td>\n",
       "      <td>Dummy output: What is some cool music from the...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>18.518519</td>\n",
       "      <td>0.100902</td>\n",
       "      <td>18.461538</td>\n",
       "      <td>15.873016</td>\n",
       "      <td>18.461538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>What are the names of some famous actors that ...</td>\n",
       "      <td>Some famous actors that started their careers ...</td>\n",
       "      <td>Dummy output: What are the names of some famou...</td>\n",
       "      <td>Dummy output: What are the names of some famou...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32.142857</td>\n",
       "      <td>5.420550</td>\n",
       "      <td>31.034483</td>\n",
       "      <td>28.571429</td>\n",
       "      <td>31.034483</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Hi, my sister and her girlfriends want me to p...</td>\n",
       "      <td>Kickball is a game similar to baseball, but wi...</td>\n",
       "      <td>Dummy output: Hi, my sister and her girlfriend...</td>\n",
       "      <td>Dummy output: Hi, my sister and her girlfriend...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>16.071429</td>\n",
       "      <td>0.773922</td>\n",
       "      <td>15.267176</td>\n",
       "      <td>3.100775</td>\n",
       "      <td>12.213740</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>How do I wrap a present neatly?</td>\n",
       "      <td>1. Start by gathering the supplies you will ne...</td>\n",
       "      <td>Dummy output: How do I wrap a present neatly?</td>\n",
       "      <td>Dummy output: How do I wrap a present neatly?</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.210526</td>\n",
       "      <td>0.000130</td>\n",
       "      <td>5.217391</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.478261</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>How did US states get their names?</td>\n",
       "      <td>US states get their names from a variety of so...</td>\n",
       "      <td>Dummy output: How did US states get their names?</td>\n",
       "      <td>Dummy output: How did US states get their names?</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.494253</td>\n",
       "      <td>0.005193</td>\n",
       "      <td>10.416667</td>\n",
       "      <td>8.510638</td>\n",
       "      <td>10.416667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            question  \\\n",
       "0            What is some cool music from the 1920s?   \n",
       "1  What are the names of some famous actors that ...   \n",
       "2  Hi, my sister and her girlfriends want me to p...   \n",
       "3                    How do I wrap a present neatly?   \n",
       "4                 How did US states get their names?   \n",
       "\n",
       "                                              answer  \\\n",
       "0  Some cool music from the 1920s includes jazz c...   \n",
       "1  Some famous actors that started their careers ...   \n",
       "2  Kickball is a game similar to baseball, but wi...   \n",
       "3  1. Start by gathering the supplies you will ne...   \n",
       "4  US states get their names from a variety of so...   \n",
       "\n",
       "                                          prediction  \\\n",
       "0  Dummy output: What is some cool music from the...   \n",
       "1  Dummy output: What are the names of some famou...   \n",
       "2  Dummy output: Hi, my sister and her girlfriend...   \n",
       "3      Dummy output: How do I wrap a present neatly?   \n",
       "4   Dummy output: How did US states get their names?   \n",
       "\n",
       "                                 filtered_prediction  exact_match         f1  \\\n",
       "0  Dummy output: What is some cool music from the...          0.0  18.518519   \n",
       "1  Dummy output: What are the names of some famou...          0.0  32.142857   \n",
       "2  Dummy output: Hi, my sister and her girlfriend...          0.0  16.071429   \n",
       "3      Dummy output: How do I wrap a present neatly?          0.0   4.210526   \n",
       "4   Dummy output: How did US states get their names?          0.0  11.494253   \n",
       "\n",
       "   sacrebleu     rouge1     rouge2     rougeL  \n",
       "0   0.100902  18.461538  15.873016  18.461538  \n",
       "1   5.420550  31.034483  28.571429  31.034483  \n",
       "2   0.773922  15.267176   3.100775  12.213740  \n",
       "3   0.000130   5.217391   0.000000   3.478261  \n",
       "4   0.005193  10.416667   8.510638  10.416667  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "pd.DataFrame(out.df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['llm_as_judge,lm_eval,alpaca-eval-5',\n",
       " 'general,lm_eval,alpaca-eval-5',\n",
       " 'custom,lm_eval,alpaca-eval-5',\n",
       " 'general,lm_eval,gpt4-10',\n",
       " 'custom,lm_eval,gpt4-10',\n",
       " 'llm_as_judge,lm_eval,gpt4-10',\n",
       " 'llm_as_judge,lm_eval,alpaca-gpt4-10',\n",
       " 'custom,lm_eval,alpaca-gpt4-10',\n",
       " 'general,lm_eval,alpaca-gpt4-10']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get all eval id with `template,app,dataset` format\n",
    "evaluator.get_eval_ids()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['exact_match', 'f1', 'sacrebleu', 'rouge1', 'rouge2', 'rougeL']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get metric names of llm_as_judge template\n",
    "evaluator.get_metric_name(\"general\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['llm_as_judge', 'truthfulqa_template_gen', 'general']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get supported templates\n",
    "evaluator.predefined_templates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "module_eval",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}