synthetic_dataset.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "## Document loading"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pprint import pp\n",
    "%reload_ext dotenv\n",
    "%dotenv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "pd.set_option(\n",
    "    \"display.max_colwidth\", None\n",
    ")"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [],
   "source": [
    "from llama_index.core import Document\n",
    "import os\n",
    "from elasticsearch import Elasticsearch\n",
    "\n",
    "def get_documents_from_es_sources():\n",
    "    client = Elasticsearch(\n",
    "        cloud_id=os.getenv(\"es_cloud_id\"),  # found within the deployment page\n",
    "        basic_auth=(os.getenv(\"es_user\"), os.getenv(\"es_password\"))\n",
    "    )\n",
    "\n",
    "    index_name = \"paper\"\n",
    "    query = {\n",
    "        \"match_all\": {}\n",
    "    }\n",
    "\n",
    "    response = client.search(index=index_name, query=query, _source=[\"content\", \"metadata\"], size=1000)\n",
    "    print(len(response['hits']['hits']))\n",
    "\n",
    "    documents = []\n",
    "\n",
    "    for hit in response['hits']['hits']:\n",
    "        doc = Document(\n",
    "            metadata=hit['_source']['metadata'],\n",
    "            text=hit['_source']['content']\n",
    "        )\n",
    "        documents.append(doc)\n",
    "    return documents"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [],
   "source": [
    "from indexing.components.loading import DocumentReader\n",
    "from indexing.components.indexing import ChunkStrategyModule\n",
    "\n",
    "def get_documents_from_simple_chunking():\n",
    "    rag_documents = DocumentReader(file_path=[\"indexing/data/papers/rag_survey.pdf\"]).pdf_reader()\n",
    "    eval_documents = DocumentReader(file_path=[\"indexing/data/papers/eval_survey.pdf\"]).pdf_reader()\n",
    "\n",
    "    documents = rag_documents + eval_documents\n",
    "\n",
    "    print(f\"Document counts: {len(documents)}\")\n",
    "\n",
    "    nodes = ChunkStrategyModule(documents=documents).base_parser(chunk_size=512, chunk_overlap=32)\n",
    "\n",
    "    print(f\"Node counts: {len(nodes)}\")\n",
    "    return nodes"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Document counts: 41\n",
      "Node counts: 112\n"
     ]
    }
   ],
   "source": [
    "documents = get_documents_from_simple_chunking()"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [],
   "source": [
    "doc_id_2_gen_questions = {}\n",
    "\n",
    "for document in documents:\n",
    "    doc_id_2_gen_questions[document.id_] = []"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Setup agents for question generation"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [],
   "source": [
    "from inferences.components.models import get_bedrock_li_text_model, get_openai_model, ModelEnum\n",
    "\n",
    "\n",
    "def call_llm(prompt: str):\n",
    "    llm = get_openai_model(\n",
    "        temperature=0.5,\n",
    "        max_tokens=1000\n",
    "    )\n",
    "\n",
    "    return llm.complete(\n",
    "        prompt\n",
    "    ).text"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "outputs": [],
   "source": [
    "# Your factoid question and answer should be unique in meaning to any of the given previous (question, answer) list.\n",
    "# Now here is the previous questions.\n",
    "# Previous questions list: {previous_questions}"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "outputs": [],
   "source": [
    "QA_generation_prompt = \"\"\"\n",
    "Your task is to write a factoid question and an answer given a context.\n",
    "Your factoid question should be answerable with a specific, concise piece of factual information from the context.\n",
    "Your factoid question should be formulated in the same style as questions users could ask in a search engine.\n",
    "This means that your factoid question MUST NOT mention something like \"according to the passage\" or \"context\".\n",
    "\n",
    "Provide your answer as follows:\n",
    "\n",
    "Output:::\n",
    "Factoid question: (your factoid question)\n",
    "Answer: (your answer to the factoid question)\n",
    "\n",
    "Now here is the context.\n",
    "\n",
    "Context: {context}\\n\n",
    "Output:::\"\"\""
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 150/150 [02:52<00:00,  1.15s/it]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "import random\n",
    "\n",
    "N_GENERATIONS = 150\n",
    "\n",
    "outputs = []\n",
    "\n",
    "for document in tqdm(random.choices(documents, k=N_GENERATIONS)):\n",
    "    output_QA_couple = call_llm(QA_generation_prompt.format(context=document.text, previous_questions=doc_id_2_gen_questions[document.id_]))\n",
    "    try:\n",
    "        question = output_QA_couple.split(\"Factoid question: \")[-1].split(\"Answer: \")[0]\n",
    "        answer = output_QA_couple.split(\"Answer: \")[-1]\n",
    "        assert len(answer) < 300, \"Answer is too long\"\n",
    "        outputs.append(\n",
    "            {\n",
    "                \"question\": question.strip(),\n",
    "                \"context\": document.text.strip(),\n",
    "                \"answer\": answer.strip()\n",
    "            }\n",
    "        )\n",
    "        doc_id_2_gen_questions[document.id_].append((question.strip(), answer.strip()))\n",
    "    except Exception:\n",
    "        continue"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [
    {
     "data": {
      "text/plain": "                                                                                                                                              question  \\\n0                                                                                                                 What is reranking in the RAG system?   \n1                                                                                              What is the main focus of the survey on RAG evaluation?   \n2                                                                                                                 What is the core task of RAG models?   \n3                                               What is the title of the paper by X. Li, R. Zhao, Y. K. Chia, B. Ding, L. Bing, S. Joty, and S. Poria?   \n4                                                             What is the title of the paper authored by S. Zhuang, B. Liu, B. Koopman, and G. Zuccon?   \n..                                                                                                                                                 ...   \n145  When was the paper \"The cot collection: Improving zero-shot and few-shot learning of language models via chain-of-thought fine-tuning\" published?   \n146                                                              When was the paper \"Best Practices for LLM Evaluation of RAG Applications\" published?   \n147                                                                                        What is the analysis framework proposed for RAG benchmarks?   \n148                                                                    What is the title of the paper with the arXiv preprint number arXiv:2310.11511?   \n149                                                           What are recent strategies, such as CRUD-based assessments, scrutinizing in RAG systems?   \n\n                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 context  \\\n0                                                                                                                                                                                                                                                                                                                                                                                                                                        10\\nintroduces an innovative method for integrating knowledge\\ninto white-box models via directive fine-tuning [75]. In this\\napproach, the retriever module is directly substituted to gen-\\nerate relevant documents according to a query. This method\\nassists in addressing the difficulties encountered during the\\nfine-tuning process and enhances model performance.\\nIV. G ENERATION\\nAfter retrieval, it is not a good practice to directly input all\\nthe retrieved information to the LLM for answering questions.\\nFollowing will introduce adjustments from two perspectives:\\nadjusting the retrieved content and adjusting the LLM.\\nA. Context Curation\\nRedundant information can interfere with the final gener-\\nation of LLM, and overly long contexts can also lead LLM\\nto the “Lost in the middle” problem [98]. Like humans, LLM\\ntends to only focus on the beginning and end of long texts,\\nwhile forgetting the middle portion. Therefore, in the RAG\\nsystem, we typically need to further process the retrieved\\ncontent.\\n1) Reranking: Reranking fundamentally reorders document\\nchunks to highlight the most pertinent results first, effectively\\nreducing the overall document pool, severing a dual purpose\\nin information retrieval, acting as both an enhancer and a\\nfilter, delivering refined inputs for more precise language\\nmodel processing [70]. Reranking can be performed using\\nrule-based methods that depend on predefined metrics like\\nDiversity, Relevance, and MRR, or model-based approaches\\nlike Encoder-Decoder models from the BERT series (e.g.,\\nSpanBERT), specialized reranking models such as Cohere\\nrerank or bge-raranker-large, and general large language mod-\\nels like GPT [12], [99].\\n2) Context Selection/Compression: A common misconcep-\\ntion in the RAG process is the belief that retrieving as many\\nrelevant documents as possible and concatenating them to form\\na lengthy retrieval prompt is beneficial. However, excessive\\ncontext can introduce more noise, diminishing the LLM’s\\nperception of key information .   \n1                         Evaluation of Retrieval-Augmented Generation: A Survey 3\\nmethods, indicators, and tools, particularly given the black-box LLM generation. Eval-\\nuating RAG systems thus involves considering quite a few specific components and\\nthe complexity of overall system assessment. On the other hand, the complexity of\\nRAG systems is further compounded by the dynamic external database and the various\\ndownstream tasks, such as content creation or open domain question answering [14,53].\\nThese challenges necessitate the development of comprehensive evaluation metrics that\\ncan effectively capture the interplay between retrieval accuracy and generative qual-\\nity [2,6]. To clarify the elements further, we conducted this survey on RAG evalua-\\ntion to address the current gaps in the area, which differs from the prior RAG surveys\\n[57,14,21] that predominantly collected specific RAG methods or data. We have com-\\npiled 12 distinct evaluation frameworks, encompassing a range of aspects of the RAG\\nsystem. We conduct a comparative analysis and synthesize the specific evaluation meth-\\nods of various components, focusing on aspects such as accuracy, faithfulness, and rel-\\nevance. We also discuss the constraints of the existing methodology and the prospects\\nfor future RAG evaluations. We hope to provide the readers with a comprehensive un-\\nderstanding of the RAG evaluation.\\nFor this paper, we contribute in the following aspects:\\n1.Challenge of Evaluation : This is the first work that summarize and classify the\\nchallenges in evaluating RAG systems through the structure of RAG systems, in-\\ncluding three parts retrieval, generation, and the whole system.\\n2.Analysis Framework : Based on the challenges, we propose an analysis framework\\n(RGAG ) for RAG benchmarks, which is designed to navigate the unique complex-\\nities inherent to RAG systems, offering a fundamental methodology for assessing\\ntheir efficacy across many facets.\\n3.RAG Benchmark Analysis : With the help of the RGAG framework, we provide a\\ncomprehensive analysis of existing RAG benchmarks, highlighting their strengths\\nand limitations and proposing recommendations for future developments in RAG\\nsystem evaluation.\\n2 Challenges in Evaluating RAG Systems\\nEvaluating hybrid RAG systems entails evaluating retrieval, generation and the RAG\\nsystem as a whole. These evaluations are multifaceted, requiring careful consideration\\nand analysis.   \n2                                                                                                                                                                                                                                                      12\\nprobability of generated terms [24]. When the probability falls\\nbelow a certain threshold would activates the retrieval system\\nto collect relevant information, thus optimizing the retrieval\\ncycle. Self-RAG [25] introduces “reflection tokens” that allow\\nthe model to introspect its outputs. These tokens come in\\ntwo varieties: “retrieve” and “critic”. The model autonomously\\ndecides when to activate retrieval, or alternatively, a predefined\\nthreshold may trigger the process. During retrieval, the gen-\\nerator conducts a fragment-level beam search across multiple\\nparagraphs to derive the most coherent sequence. Critic scores\\nare used to update the subdivision scores, with the flexibility\\nto adjust these weights during inference, tailoring the model’s\\nbehavior. Self-RAG’s design obviates the need for additional\\nclassifiers or reliance on Natural Language Inference (NLI)\\nmodels, thus streamlining the decision-making process for\\nwhen to engage retrieval mechanisms and improving the\\nmodel’s autonomous judgment capabilities in generating ac-\\ncurate responses.\\nVI. T ASK AND EVALUATION\\nThe rapid advancement and growing adoption of RAG\\nin the field of NLP have propelled the evaluation of RAG\\nmodels to the forefront of research in the LLMs community.\\nThe primary objective of this evaluation is to comprehend\\nand optimize the performance of RAG models across diverse\\napplication scenarios.This chapter will mainly introduce the\\nmain downstream tasks of RAG, datasets, and how to evaluate\\nRAG systems.\\nA. Downstream Task\\nThe core task of RAG remains Question Answering (QA),\\nincluding traditional single-hop/multi-hop QA, multiple-\\nchoice, domain-specific QA as well as long-form scenarios\\nsuitable for RAG. In addition to QA, RAG is continuously\\nbeing expanded into multiple downstream tasks, such as Infor-\\nmation Extraction (IE), dialogue generation, code search, etc.\\nThe main downstream tasks of RAG and their corresponding\\ndatasets are summarized in Table II.\\nB. Evaluation Target\\nHistorically, RAG models assessments have centered on\\ntheir execution in specific downstream tasks. These evaluations\\nemploy established metrics suitable to the tasks at hand.   \n3                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      Yih, “Dense passage retrieval for open-domain question\\nanswering,” arXiv preprint arXiv:2004.04906 , 2020.\\n[103] Y . Ma, Y . Cao, Y . Hong, and A. Sun, “Large language model is\\nnot a good few-shot information extractor, but a good reranker for\\nhard samples!” ArXiv , vol. abs/2303.08559, 2023. [Online]. Available:\\nhttps://api.semanticscholar.org/CorpusID:257532405\\n[104] J. Cui, Z. Li, Y . Yan, B. Chen, and L. Yuan, “Chatlaw: Open-source\\nlegal large language model with integrated external knowledge bases,”\\narXiv preprint arXiv:2306.16092 , 2023.\\n[105] O. Yoran, T. Wolfson, O. Ram, and J. Berant, “Making retrieval-\\naugmented language models robust to irrelevant context,” arXiv\\npreprint arXiv:2310.01558 , 2023.\\n[106] X. Li, R. Zhao, Y . K. Chia, B. Ding, L. Bing, S. Joty, and S. Poria,\\n“Chain of knowledge: A framework for grounding large language mod-\\nels with structured knowledge bases,” arXiv preprint arXiv:2305.13269 ,\\n2023.\\n[107] H. Yang, S. Yue, and Y . He, “Auto-gpt for online decision\\nmaking: Benchmarks and additional opinions,” arXiv preprint\\narXiv:2306.02224 , 2023.\\n[108] T. Schick, J. Dwivedi-Yu, R. Dess `ı, R. Raileanu, M. Lomeli, L. Zettle-\\nmoyer, N. Cancedda, and T. Scialom, “Toolformer: Language models\\ncan teach themselves to use tools,” arXiv preprint arXiv:2302.04761 ,\\n2023.   \n4                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             Ling, “Corrective retrieval\\naugmented generation,” arXiv preprint arXiv:2401.15884 , 2024.\\n[68] P. Jain, L. B. Soares, and T. Kwiatkowski, “1-pager: One pass answer\\ngeneration and evidence retrieval,” arXiv preprint arXiv:2310.16568 ,\\n2023.\\n[69] H. Yang, Z. Li, Y . Zhang, J. Wang, N. Cheng, M. Li, and J. Xiao, “Prca:\\nFitting black-box large language models for retrieval question answer-\\ning via pluggable reward-driven contextual adapter,” arXiv preprint\\narXiv:2310.18347 , 2023.\\n[70] S. Zhuang, B. Liu, B. Koopman, and G. Zuccon, “Open-source large\\nlanguage models are strong zero-shot query likelihood models for\\ndocument ranking,” arXiv preprint arXiv:2310.13243 , 2023.\\n[71] F. Xu, W. Shi, and E. Choi, “Recomp: Improving retrieval-augmented\\nlms with compression and selective augmentation,” arXiv preprint\\narXiv:2310.04408 , 2023.\\n[72] W. Shi, S. Min, M. Yasunaga, M. Seo, R. James, M. Lewis, L. Zettle-\\nmoyer, and W.-t. Yih, “Replug: Retrieval-augmented black-box lan-\\nguage models,” arXiv preprint arXiv:2301.12652 , 2023.\\n[73] E. Melz, “Enhancing llm intelligence with arm-rag: Auxiliary ra-\\ntionale memory for retrieval augmented generation,” arXiv preprint\\narXiv:2311.04177 , 2023.\\n[74] H. Wang, W. Huang, Y . Deng, R. Wang, Z. Wang, Y . Wang, F. Mi,\\nJ. Z. Pan, and K.-F.   \n..                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   ...   \n145                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            [142] O. Levy, M. Seo, E. Choi, and L. Zettlemoyer, “Zero-shot relation ex-\\ntraction via reading comprehension,” arXiv preprint arXiv:1706.04115 ,\\n2017.\\n[143] R. Zellers, A. Holtzman, Y . Bisk, A. Farhadi, and Y . Choi, “Hel-\\nlaswag: Can a machine really finish your sentence?” arXiv preprint\\narXiv:1905.07830 , 2019.\\n[144] S. Kim, S. J. Joo, D. Kim, J. Jang, S. Ye, J. Shin, and M. Seo,\\n“The cot collection: Improving zero-shot and few-shot learning of\\nlanguage models via chain-of-thought fine-tuning,” arXiv preprint\\narXiv:2305.14045 , 2023.\\n[145] A. Saha, V . Pahuja, M. Khapra, K. Sankaranarayanan, and S. Chandar,\\n“Complex sequential question answering: Towards learning to converse\\nover linked question answer pairs with a knowledge graph,” in Proceed-\\nings of the AAAI conference on artificial intelligence , vol. 32, no. 1,\\n2018.\\n[146] D. Hendrycks, C. Burns, S. Basart, A. Zou, M. Mazeika, D. Song, and\\nJ. Steinhardt, “Measuring massive multitask language understanding,”\\narXiv preprint arXiv:2009.03300 , 2020.\\n[147] S. Merity, C. Xiong, J. Bradbury, and R. Socher, “Pointer sentinel\\nmixture models,” arXiv preprint arXiv:1609.07843 , 2016.\\n[148] M. Geva, D. Khashabi, E. Segal, T. Khot, D. Roth, and J. Berant,\\n“Did aristotle use a laptop? a question answering benchmark with\\nimplicit reasoning strategies,” Transactions of the Association for\\nComputational Linguistics , vol. 9, pp. 346–361, 2021.   \n146                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   Transactions of the Association for Computational Linguistics 7, 453–\\n466 (2019). https://doi.org/10.1162/tacl_a_00276 ,https://doi.org/\\n10.1162/tacl_a_00276\\n26. Lanchantin, J., Toshniwal, S., Weston, J., Szlam, A., Sukhbaatar, S.: Learning to reason\\nand memorize with self-notes (May 2023). https://doi.org/10.48550/ARXIV.\\n2305.00833\\n27. LangChain: Evaluating rag architectures on benchmark tasks (Nov 2023), https:\\n//langchain-ai.github.io/langchain-benchmarks/notebooks/\\nretrieval/langchain_docs_qa.html\\n28. Leng, Q., Uhlenhuth, K., Polyzotis, A.: Best Practices for LLM Evaluation of\\nRAG Applications (Dec 2023), https://www.databricks.com/blog/\\nLLM-auto-eval-best-practices-RAG\\n29. Lewis, P., Perez, E., Piktus, A., Petroni, F., Karpukhin, V ., Goyal, N., Küttler, H., Lewis,\\nM., Yih, W.t., Rocktäschel, T., Riedel, S., Kiela, D.: Retrieval-augmented generation for\\nknowledge-intensive NLP tasks. In: Proceedings of the 34th International Conference on\\nNeural Information Processing Systems. pp. 9459–9474. NIPS’20, Curran Associates Inc.,\\nRed Hook, NY , USA (Dec 2020)   \n147                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              Diverse and compre-\\nhensive datasets that accurately reflect real-world scenarios are crucial. Challenges also\\narise in the realm of metrics, encompassing generative evaluation criteria for distinct\\ndownstream tasks, human preferences, and practical considerations within the RAG\\nsystem. Most prior benchmarks predominantly tackle one or several aspects of the RAG\\nassessment but lack a comprehensive, holistic analysis.\\nTo provide a better understanding of RAG benchmarks, we propose an analysis\\nframework named RGAR (Retrieval, Generation, and Additional Requirement). It takes\\ninto account the Target ,Dataset , and Metric respectively. The Target module is in-\\ntended to determine the evaluation direction. The Dataset module facilitates the com-\\nparison of various data constructions in RAG benchmarks. The final module, Metrics,\\nintroduces the metrics that correspond to specific targets and datasets used during eval-\\nuation. Overall, it is designed to provide a systematic methodology for assessing the\\neffectiveness of RAG systems across various aspects by covering all possible pairs be-\\ntween the “Evaluable Outputs” (EOs) and “Ground Truths” (GTs). In the following   \n148                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                Yang,\\nJ. Callan, and G. Neubig, “Active retrieval augmented generation,”\\narXiv preprint arXiv:2305.06983 , 2023.\\n[25] A. Asai, Z. Wu, Y . Wang, A. Sil, and H. Hajishirzi, “Self-rag:\\nLearning to retrieve, generate, and critique through self-reflection,”\\narXiv preprint arXiv:2310.11511 , 2023.\\n[26] Z. Ke, W. Kong, C. Li, M. Zhang, Q. Mei, and M. Bendersky,\\n“Bridging the preference gap between retrievers and llms,” arXiv\\npreprint arXiv:2401.06954 , 2024.\\n[27] X. V . Lin, X. Chen, M. Chen, W. Shi, M. Lomeli, R. James, P. Ro-\\ndriguez, J. Kahn, G. Szilvasy, M. Lewis et al. , “Ra-dit: Retrieval-\\naugmented dual instruction tuning,” arXiv preprint arXiv:2310.01352 ,\\n2023.\\n[28] O. Ovadia, M. Brief, M. Mishaeli, and O. Elisha, “Fine-tuning or\\nretrieval? comparing knowledge injection in llms,” arXiv preprint\\narXiv:2312.05934 , 2023.\\n[29] T. Lan, D. Cai, Y . Wang, H. Huang, and X.-L. Mao, “Copy is all\\nyou need,” in The Eleventh International Conference on Learning\\nRepresentations , 2022.\\n[30] T. Chen, H. Wang, S. Chen, W. Yu, K. Ma, X. Zhao, D. Yu, and\\nH. Zhang, “Dense x retrieval: What retrieval granularity should we\\nuse?” arXiv preprint arXiv:2312.06648 , 2023.   \n149  Evaluation of Retrieval-Augmented Generation: A Survey 13\\nwell they align with human preferences for clarity, relevance, and factual accuracy.\\nHowever, recent strategies, such as CRUD-based assessments, offer novel angles by\\nscrutinizing RAG systems’ interactive capabilities with dynamic information environ-\\nments [34]. These methodologies underscore the necessity for RAG evaluations to\\nevolve beyond static benchmarks, mirroring real-world scenarios where information\\nis continuously updated and queries are not strictly fact-based but exploratory or con-\\nversational.\\nOn the dataset front, the challenge of devising a \"one-size-fits-all\" dataset is pro-\\nnounced, given the highly task-specific nature of RAG systems. Unique datasets, metic-\\nulously crafted to test specific facets of RAG performance, are indispensable. While\\nthis approach ensures thorough, targeted evaluation, it also magnifies the effort and re-\\nsources required for comprehensive testing. The divergence of datasets, ranging from\\nnews articles to structured databases, reflects the adaptability required of RAG systems\\nbut also signifies a formidable barrier to streamlined evaluation [49,50].\\nWhen it comes to metrics , the use of LLMs as automatic evaluative judges signifies a\\nburgeoning trend, promising versatility and depth in generative outputs with reasoning\\non a large scale compared to human evaluation. However, using LLMs as judges for\\nchatbot responses presents challenges in aligning with human judgment, establishing\\neffective grading scales, and applying consistent evaluation across varied use cases. The\\ndetermination of correctness, clarity, and richness can differ between automated and\\nhuman assessments. Moreover, the effectiveness of example-based scoring can vary,\\nand there’s no universally applicable grading scale, complicating the standardization of\\nLLM as a judge. [28]\\nFuture directions in RAG evaluation should focus on developing more adaptive,\\ncontext-aware benchmarks that accurately reflect the dynamic, information-rich envi-\\nronments these systems are designed to navigate. Such efforts could include simulat-\\ning real-time information updates in evaluation datasets or incorporating user feedback\\nloops into assessment methodologies. Additionally, exploring more nuanced metrics\\nthat can capture the subtlety of human language comprehension and generation—beyond\\nsheer accuracy or relevance—will be crucial.   \n\n                                                                                                                                                         answer  \n0               Reranking fundamentally reorders document chunks to highlight the most pertinent results first, effectively reducing the overall document pool.  \n1    The main focus of the survey on RAG evaluation is to address the current gaps in the area and provide a comprehensive understanding of the RAG evaluation.  \n2                                                                                                       The core task of RAG models is Question Answering (QA).  \n3                                                           Chain of knowledge: A framework for grounding large language models with structured knowledge bases  \n4                                                         \"Open-source large language models are strong zero-shot query likelihood models for document ranking\"  \n..                                                                                                                                                          ...  \n145                                                                                                                                                        2023  \n146                                                                                                                                               December 2023  \n147                                                                                                    RGAR (Retrieval, Generation, and Additional Requirement)  \n148                                                                              Self-rag: Learning to retrieve, generate, and critique through self-reflection  \n149                                                                                         The interactive capabilities with dynamic information environments.  \n\n[150 rows x 3 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>question</th>\n      <th>context</th>\n      <th>answer</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>What is reranking in the RAG system?</td>\n      <td>10\\nintroduces an innovative method for integrating knowledge\\ninto white-box models via directive fine-tuning [75]. In this\\napproach, the retriever module is directly substituted to gen-\\nerate relevant documents according to a query. This method\\nassists in addressing the difficulties encountered during the\\nfine-tuning process and enhances model performance.\\nIV. G ENERATION\\nAfter retrieval, it is not a good practice to directly input all\\nthe retrieved information to the LLM for answering questions.\\nFollowing will introduce adjustments from two perspectives:\\nadjusting the retrieved content and adjusting the LLM.\\nA. Context Curation\\nRedundant information can interfere with the final gener-\\nation of LLM, and overly long contexts can also lead LLM\\nto the “Lost in the middle” problem [98]. Like humans, LLM\\ntends to only focus on the beginning and end of long texts,\\nwhile forgetting the middle portion. Therefore, in the RAG\\nsystem, we typically need to further process the retrieved\\ncontent.\\n1) Reranking: Reranking fundamentally reorders document\\nchunks to highlight the most pertinent results first, effectively\\nreducing the overall document pool, severing a dual purpose\\nin information retrieval, acting as both an enhancer and a\\nfilter, delivering refined inputs for more precise language\\nmodel processing [70]. Reranking can be performed using\\nrule-based methods that depend on predefined metrics like\\nDiversity, Relevance, and MRR, or model-based approaches\\nlike Encoder-Decoder models from the BERT series (e.g.,\\nSpanBERT), specialized reranking models such as Cohere\\nrerank or bge-raranker-large, and general large language mod-\\nels like GPT [12], [99].\\n2) Context Selection/Compression: A common misconcep-\\ntion in the RAG process is the belief that retrieving as many\\nrelevant documents as possible and concatenating them to form\\na lengthy retrieval prompt is beneficial. However, excessive\\ncontext can introduce more noise, diminishing the LLM’s\\nperception of key information .</td>\n      <td>Reranking fundamentally reorders document chunks to highlight the most pertinent results first, effectively reducing the overall document pool.</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>What is the main focus of the survey on RAG evaluation?</td>\n      <td>Evaluation of Retrieval-Augmented Generation: A Survey 3\\nmethods, indicators, and tools, particularly given the black-box LLM generation. Eval-\\nuating RAG systems thus involves considering quite a few specific components and\\nthe complexity of overall system assessment. On the other hand, the complexity of\\nRAG systems is further compounded by the dynamic external database and the various\\ndownstream tasks, such as content creation or open domain question answering [14,53].\\nThese challenges necessitate the development of comprehensive evaluation metrics that\\ncan effectively capture the interplay between retrieval accuracy and generative qual-\\nity [2,6]. To clarify the elements further, we conducted this survey on RAG evalua-\\ntion to address the current gaps in the area, which differs from the prior RAG surveys\\n[57,14,21] that predominantly collected specific RAG methods or data. We have com-\\npiled 12 distinct evaluation frameworks, encompassing a range of aspects of the RAG\\nsystem. We conduct a comparative analysis and synthesize the specific evaluation meth-\\nods of various components, focusing on aspects such as accuracy, faithfulness, and rel-\\nevance. We also discuss the constraints of the existing methodology and the prospects\\nfor future RAG evaluations. We hope to provide the readers with a comprehensive un-\\nderstanding of the RAG evaluation.\\nFor this paper, we contribute in the following aspects:\\n1.Challenge of Evaluation : This is the first work that summarize and classify the\\nchallenges in evaluating RAG systems through the structure of RAG systems, in-\\ncluding three parts retrieval, generation, and the whole system.\\n2.Analysis Framework : Based on the challenges, we propose an analysis framework\\n(RGAG ) for RAG benchmarks, which is designed to navigate the unique complex-\\nities inherent to RAG systems, offering a fundamental methodology for assessing\\ntheir efficacy across many facets.\\n3.RAG Benchmark Analysis : With the help of the RGAG framework, we provide a\\ncomprehensive analysis of existing RAG benchmarks, highlighting their strengths\\nand limitations and proposing recommendations for future developments in RAG\\nsystem evaluation.\\n2 Challenges in Evaluating RAG Systems\\nEvaluating hybrid RAG systems entails evaluating retrieval, generation and the RAG\\nsystem as a whole. These evaluations are multifaceted, requiring careful consideration\\nand analysis.</td>\n      <td>The main focus of the survey on RAG evaluation is to address the current gaps in the area and provide a comprehensive understanding of the RAG evaluation.</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>What is the core task of RAG models?</td>\n      <td>12\\nprobability of generated terms [24]. When the probability falls\\nbelow a certain threshold would activates the retrieval system\\nto collect relevant information, thus optimizing the retrieval\\ncycle. Self-RAG [25] introduces “reflection tokens” that allow\\nthe model to introspect its outputs. These tokens come in\\ntwo varieties: “retrieve” and “critic”. The model autonomously\\ndecides when to activate retrieval, or alternatively, a predefined\\nthreshold may trigger the process. During retrieval, the gen-\\nerator conducts a fragment-level beam search across multiple\\nparagraphs to derive the most coherent sequence. Critic scores\\nare used to update the subdivision scores, with the flexibility\\nto adjust these weights during inference, tailoring the model’s\\nbehavior. Self-RAG’s design obviates the need for additional\\nclassifiers or reliance on Natural Language Inference (NLI)\\nmodels, thus streamlining the decision-making process for\\nwhen to engage retrieval mechanisms and improving the\\nmodel’s autonomous judgment capabilities in generating ac-\\ncurate responses.\\nVI. T ASK AND EVALUATION\\nThe rapid advancement and growing adoption of RAG\\nin the field of NLP have propelled the evaluation of RAG\\nmodels to the forefront of research in the LLMs community.\\nThe primary objective of this evaluation is to comprehend\\nand optimize the performance of RAG models across diverse\\napplication scenarios.This chapter will mainly introduce the\\nmain downstream tasks of RAG, datasets, and how to evaluate\\nRAG systems.\\nA. Downstream Task\\nThe core task of RAG remains Question Answering (QA),\\nincluding traditional single-hop/multi-hop QA, multiple-\\nchoice, domain-specific QA as well as long-form scenarios\\nsuitable for RAG. In addition to QA, RAG is continuously\\nbeing expanded into multiple downstream tasks, such as Infor-\\nmation Extraction (IE), dialogue generation, code search, etc.\\nThe main downstream tasks of RAG and their corresponding\\ndatasets are summarized in Table II.\\nB. Evaluation Target\\nHistorically, RAG models assessments have centered on\\ntheir execution in specific downstream tasks. These evaluations\\nemploy established metrics suitable to the tasks at hand.</td>\n      <td>The core task of RAG models is Question Answering (QA).</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>What is the title of the paper by X. Li, R. Zhao, Y. K. Chia, B. Ding, L. Bing, S. Joty, and S. Poria?</td>\n      <td>Yih, “Dense passage retrieval for open-domain question\\nanswering,” arXiv preprint arXiv:2004.04906 , 2020.\\n[103] Y . Ma, Y . Cao, Y . Hong, and A. Sun, “Large language model is\\nnot a good few-shot information extractor, but a good reranker for\\nhard samples!” ArXiv , vol. abs/2303.08559, 2023. [Online]. Available:\\nhttps://api.semanticscholar.org/CorpusID:257532405\\n[104] J. Cui, Z. Li, Y . Yan, B. Chen, and L. Yuan, “Chatlaw: Open-source\\nlegal large language model with integrated external knowledge bases,”\\narXiv preprint arXiv:2306.16092 , 2023.\\n[105] O. Yoran, T. Wolfson, O. Ram, and J. Berant, “Making retrieval-\\naugmented language models robust to irrelevant context,” arXiv\\npreprint arXiv:2310.01558 , 2023.\\n[106] X. Li, R. Zhao, Y . K. Chia, B. Ding, L. Bing, S. Joty, and S. Poria,\\n“Chain of knowledge: A framework for grounding large language mod-\\nels with structured knowledge bases,” arXiv preprint arXiv:2305.13269 ,\\n2023.\\n[107] H. Yang, S. Yue, and Y . He, “Auto-gpt for online decision\\nmaking: Benchmarks and additional opinions,” arXiv preprint\\narXiv:2306.02224 , 2023.\\n[108] T. Schick, J. Dwivedi-Yu, R. Dess `ı, R. Raileanu, M. Lomeli, L. Zettle-\\nmoyer, N. Cancedda, and T. Scialom, “Toolformer: Language models\\ncan teach themselves to use tools,” arXiv preprint arXiv:2302.04761 ,\\n2023.</td>\n      <td>Chain of knowledge: A framework for grounding large language models with structured knowledge bases</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>What is the title of the paper authored by S. Zhuang, B. Liu, B. Koopman, and G. Zuccon?</td>\n      <td>Ling, “Corrective retrieval\\naugmented generation,” arXiv preprint arXiv:2401.15884 , 2024.\\n[68] P. Jain, L. B. Soares, and T. Kwiatkowski, “1-pager: One pass answer\\ngeneration and evidence retrieval,” arXiv preprint arXiv:2310.16568 ,\\n2023.\\n[69] H. Yang, Z. Li, Y . Zhang, J. Wang, N. Cheng, M. Li, and J. Xiao, “Prca:\\nFitting black-box large language models for retrieval question answer-\\ning via pluggable reward-driven contextual adapter,” arXiv preprint\\narXiv:2310.18347 , 2023.\\n[70] S. Zhuang, B. Liu, B. Koopman, and G. Zuccon, “Open-source large\\nlanguage models are strong zero-shot query likelihood models for\\ndocument ranking,” arXiv preprint arXiv:2310.13243 , 2023.\\n[71] F. Xu, W. Shi, and E. Choi, “Recomp: Improving retrieval-augmented\\nlms with compression and selective augmentation,” arXiv preprint\\narXiv:2310.04408 , 2023.\\n[72] W. Shi, S. Min, M. Yasunaga, M. Seo, R. James, M. Lewis, L. Zettle-\\nmoyer, and W.-t. Yih, “Replug: Retrieval-augmented black-box lan-\\nguage models,” arXiv preprint arXiv:2301.12652 , 2023.\\n[73] E. Melz, “Enhancing llm intelligence with arm-rag: Auxiliary ra-\\ntionale memory for retrieval augmented generation,” arXiv preprint\\narXiv:2311.04177 , 2023.\\n[74] H. Wang, W. Huang, Y . Deng, R. Wang, Z. Wang, Y . Wang, F. Mi,\\nJ. Z. Pan, and K.-F.</td>\n      <td>\"Open-source large language models are strong zero-shot query likelihood models for document ranking\"</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>145</th>\n      <td>When was the paper \"The cot collection: Improving zero-shot and few-shot learning of language models via chain-of-thought fine-tuning\" published?</td>\n      <td>[142] O. Levy, M. Seo, E. Choi, and L. Zettlemoyer, “Zero-shot relation ex-\\ntraction via reading comprehension,” arXiv preprint arXiv:1706.04115 ,\\n2017.\\n[143] R. Zellers, A. Holtzman, Y . Bisk, A. Farhadi, and Y . Choi, “Hel-\\nlaswag: Can a machine really finish your sentence?” arXiv preprint\\narXiv:1905.07830 , 2019.\\n[144] S. Kim, S. J. Joo, D. Kim, J. Jang, S. Ye, J. Shin, and M. Seo,\\n“The cot collection: Improving zero-shot and few-shot learning of\\nlanguage models via chain-of-thought fine-tuning,” arXiv preprint\\narXiv:2305.14045 , 2023.\\n[145] A. Saha, V . Pahuja, M. Khapra, K. Sankaranarayanan, and S. Chandar,\\n“Complex sequential question answering: Towards learning to converse\\nover linked question answer pairs with a knowledge graph,” in Proceed-\\nings of the AAAI conference on artificial intelligence , vol. 32, no. 1,\\n2018.\\n[146] D. Hendrycks, C. Burns, S. Basart, A. Zou, M. Mazeika, D. Song, and\\nJ. Steinhardt, “Measuring massive multitask language understanding,”\\narXiv preprint arXiv:2009.03300 , 2020.\\n[147] S. Merity, C. Xiong, J. Bradbury, and R. Socher, “Pointer sentinel\\nmixture models,” arXiv preprint arXiv:1609.07843 , 2016.\\n[148] M. Geva, D. Khashabi, E. Segal, T. Khot, D. Roth, and J. Berant,\\n“Did aristotle use a laptop? a question answering benchmark with\\nimplicit reasoning strategies,” Transactions of the Association for\\nComputational Linguistics , vol. 9, pp. 346–361, 2021.</td>\n      <td>2023</td>\n    </tr>\n    <tr>\n      <th>146</th>\n      <td>When was the paper \"Best Practices for LLM Evaluation of RAG Applications\" published?</td>\n      <td>Transactions of the Association for Computational Linguistics 7, 453–\\n466 (2019). https://doi.org/10.1162/tacl_a_00276 ,https://doi.org/\\n10.1162/tacl_a_00276\\n26. Lanchantin, J., Toshniwal, S., Weston, J., Szlam, A., Sukhbaatar, S.: Learning to reason\\nand memorize with self-notes (May 2023). https://doi.org/10.48550/ARXIV.\\n2305.00833\\n27. LangChain: Evaluating rag architectures on benchmark tasks (Nov 2023), https:\\n//langchain-ai.github.io/langchain-benchmarks/notebooks/\\nretrieval/langchain_docs_qa.html\\n28. Leng, Q., Uhlenhuth, K., Polyzotis, A.: Best Practices for LLM Evaluation of\\nRAG Applications (Dec 2023), https://www.databricks.com/blog/\\nLLM-auto-eval-best-practices-RAG\\n29. Lewis, P., Perez, E., Piktus, A., Petroni, F., Karpukhin, V ., Goyal, N., Küttler, H., Lewis,\\nM., Yih, W.t., Rocktäschel, T., Riedel, S., Kiela, D.: Retrieval-augmented generation for\\nknowledge-intensive NLP tasks. In: Proceedings of the 34th International Conference on\\nNeural Information Processing Systems. pp. 9459–9474. NIPS’20, Curran Associates Inc.,\\nRed Hook, NY , USA (Dec 2020)</td>\n      <td>December 2023</td>\n    </tr>\n    <tr>\n      <th>147</th>\n      <td>What is the analysis framework proposed for RAG benchmarks?</td>\n      <td>Diverse and compre-\\nhensive datasets that accurately reflect real-world scenarios are crucial. Challenges also\\narise in the realm of metrics, encompassing generative evaluation criteria for distinct\\ndownstream tasks, human preferences, and practical considerations within the RAG\\nsystem. Most prior benchmarks predominantly tackle one or several aspects of the RAG\\nassessment but lack a comprehensive, holistic analysis.\\nTo provide a better understanding of RAG benchmarks, we propose an analysis\\nframework named RGAR (Retrieval, Generation, and Additional Requirement). It takes\\ninto account the Target ,Dataset , and Metric respectively. The Target module is in-\\ntended to determine the evaluation direction. The Dataset module facilitates the com-\\nparison of various data constructions in RAG benchmarks. The final module, Metrics,\\nintroduces the metrics that correspond to specific targets and datasets used during eval-\\nuation. Overall, it is designed to provide a systematic methodology for assessing the\\neffectiveness of RAG systems across various aspects by covering all possible pairs be-\\ntween the “Evaluable Outputs” (EOs) and “Ground Truths” (GTs). In the following</td>\n      <td>RGAR (Retrieval, Generation, and Additional Requirement)</td>\n    </tr>\n    <tr>\n      <th>148</th>\n      <td>What is the title of the paper with the arXiv preprint number arXiv:2310.11511?</td>\n      <td>Yang,\\nJ. Callan, and G. Neubig, “Active retrieval augmented generation,”\\narXiv preprint arXiv:2305.06983 , 2023.\\n[25] A. Asai, Z. Wu, Y . Wang, A. Sil, and H. Hajishirzi, “Self-rag:\\nLearning to retrieve, generate, and critique through self-reflection,”\\narXiv preprint arXiv:2310.11511 , 2023.\\n[26] Z. Ke, W. Kong, C. Li, M. Zhang, Q. Mei, and M. Bendersky,\\n“Bridging the preference gap between retrievers and llms,” arXiv\\npreprint arXiv:2401.06954 , 2024.\\n[27] X. V . Lin, X. Chen, M. Chen, W. Shi, M. Lomeli, R. James, P. Ro-\\ndriguez, J. Kahn, G. Szilvasy, M. Lewis et al. , “Ra-dit: Retrieval-\\naugmented dual instruction tuning,” arXiv preprint arXiv:2310.01352 ,\\n2023.\\n[28] O. Ovadia, M. Brief, M. Mishaeli, and O. Elisha, “Fine-tuning or\\nretrieval? comparing knowledge injection in llms,” arXiv preprint\\narXiv:2312.05934 , 2023.\\n[29] T. Lan, D. Cai, Y . Wang, H. Huang, and X.-L. Mao, “Copy is all\\nyou need,” in The Eleventh International Conference on Learning\\nRepresentations , 2022.\\n[30] T. Chen, H. Wang, S. Chen, W. Yu, K. Ma, X. Zhao, D. Yu, and\\nH. Zhang, “Dense x retrieval: What retrieval granularity should we\\nuse?” arXiv preprint arXiv:2312.06648 , 2023.</td>\n      <td>Self-rag: Learning to retrieve, generate, and critique through self-reflection</td>\n    </tr>\n    <tr>\n      <th>149</th>\n      <td>What are recent strategies, such as CRUD-based assessments, scrutinizing in RAG systems?</td>\n      <td>Evaluation of Retrieval-Augmented Generation: A Survey 13\\nwell they align with human preferences for clarity, relevance, and factual accuracy.\\nHowever, recent strategies, such as CRUD-based assessments, offer novel angles by\\nscrutinizing RAG systems’ interactive capabilities with dynamic information environ-\\nments [34]. These methodologies underscore the necessity for RAG evaluations to\\nevolve beyond static benchmarks, mirroring real-world scenarios where information\\nis continuously updated and queries are not strictly fact-based but exploratory or con-\\nversational.\\nOn the dataset front, the challenge of devising a \"one-size-fits-all\" dataset is pro-\\nnounced, given the highly task-specific nature of RAG systems. Unique datasets, metic-\\nulously crafted to test specific facets of RAG performance, are indispensable. While\\nthis approach ensures thorough, targeted evaluation, it also magnifies the effort and re-\\nsources required for comprehensive testing. The divergence of datasets, ranging from\\nnews articles to structured databases, reflects the adaptability required of RAG systems\\nbut also signifies a formidable barrier to streamlined evaluation [49,50].\\nWhen it comes to metrics , the use of LLMs as automatic evaluative judges signifies a\\nburgeoning trend, promising versatility and depth in generative outputs with reasoning\\non a large scale compared to human evaluation. However, using LLMs as judges for\\nchatbot responses presents challenges in aligning with human judgment, establishing\\neffective grading scales, and applying consistent evaluation across varied use cases. The\\ndetermination of correctness, clarity, and richness can differ between automated and\\nhuman assessments. Moreover, the effectiveness of example-based scoring can vary,\\nand there’s no universally applicable grading scale, complicating the standardization of\\nLLM as a judge. [28]\\nFuture directions in RAG evaluation should focus on developing more adaptive,\\ncontext-aware benchmarks that accurately reflect the dynamic, information-rich envi-\\nronments these systems are designed to navigate. Such efforts could include simulat-\\ning real-time information updates in evaluation datasets or incorporating user feedback\\nloops into assessment methodologies. Additionally, exploring more nuanced metrics\\nthat can capture the subtlety of human language comprehension and generation—beyond\\nsheer accuracy or relevance—will be crucial.</td>\n      <td>The interactive capabilities with dynamic information environments.</td>\n    </tr>\n  </tbody>\n</table>\n<p>150 rows × 3 columns</p>\n</div>"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(pd.DataFrame(outputs))"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "outputs": [
    {
     "data": {
      "text/plain": "{'2cccb30f-7b79-427c-81c2-1e224650cdb4': [],\n '34899a27-1a74-4629-9004-a28103f310f7': [('What technology enhances language models by retrieving relevant document chunks from an external knowledge base through semantic similarity calculation?',\n   'Retrieval-Augmented Generation (RAG)'),\n  ('What technology enhances LLMs by retrieving relevant document chunks from external knowledge bases through semantic similarity calculation?',\n   'Retrieval-Augmented Generation (RAG)')],\n '208d02e1-dbf9-4268-ba40-60da18c11304': [('What are the core stages analyzed in the paper in relation to RAG?',\n   '\"Retrieval,\" \"Generation,\" and \"Augmentation\" stages are analyzed in the paper in relation to RAG.')],\n '7c934bad-f6d1-4909-bb8f-15e91fd7421b': [('What are the main components integral to the RAG process?',\n   '\"Retrieval\", \"Generation\", and \"Augmentation\"'),\n  ('What are the three main components integral to the RAG process?',\n   '\"Retrieval\", \"Generation\", and \"Augmentation\" are the three main components integral to the RAG process.')],\n 'f30e072b-6d9b-478b-a609-4d09705e65ec': [('What are the three stages of the RAG research paradigm?',\n   'Naive RAG, Advanced RAG, and Modular RAG.'),\n  ('What are the three stages of the RAG research paradigm?',\n   'Naive RAG, Advanced RAG, and Modular RAG')],\n '464699b1-bc28-434f-84fe-2fa762dae326': [],\n '0176316c-6fba-4a9b-8dce-f91e36dafb2c': [('What are some of the improvements introduced in Advanced RAG to overcome the limitations of Naive RAG?',\n   'Advanced RAG employs pre-retrieval and post-retrieval strategies, refines indexing techniques, uses a sliding window approach, fine-grained segmentation, incorporates metadata, and utilizes optimization methods.'),\n  ('What are some improvements introduced by Advanced RAG to overcome the limitations of Naive RAG?',\n   'Advanced RAG introduces specific improvements such as pre-retrieval and post-retrieval strategies, refining indexing techniques, and incorporating optimization methods to enhance retrieval quality.'),\n  ('What improvements does Advanced RAG introduce to overcome the limitations of Naive RAG?',\n   'Advanced RAG introduces specific improvements such as pre-retrieval and post-retrieval strategies, refining indexing techniques, and incorporating optimization methods to enhance retrieval quality.'),\n  ('What are some of the strategies employed by Advanced RAG to enhance retrieval quality?',\n   'Advanced RAG employs pre-retrieval and post-retrieval strategies, refines indexing techniques through a sliding window approach, fine-grained segmentation, and the incorporation of metadata, and incorporates optimization methods to streamline the retrieval process.')],\n 'd2dd637e-ff8f-4066-ae47-25eba23049b3': [('What are the main methods in the post-retrieval process of RAG?',\n   'Re-rank chunks and context compressing.'),\n  ('What are some common methods for query optimization in the pre-retrieval process of RAG?',\n   'query rewriting, query transformation, query expansion')],\n '5d6abe98-a61b-458c-9536-67ed0f32d1cb': [('What is the purpose of the Memory module in the Modular RAG framework?',\n   'The Memory module leverages the LLM’s memory to guide retrieval, creating an unbounded memory pool that.')],\n 'a754f6a4-6a21-4f07-b165-40f592490ac7': [('What module in the RAG system aims to reduce redundancy and noise by generating context directly through the LLM?',\n   'The Predict module.'),\n  ('What does the Predict module aim to reduce in the RAG system?',\n   'redundancy and noise')],\n '6616a4a8-2343-4858-a59c-5bfccd839f0d': [('What method is likened to providing a model with a tailored textbook for information retrieval?',\n   'RAG'),\n  ('What method is often compared with RAG in the context of LLM optimization?',\n   'Fine-tuning (FT)'),\n  ('What method is likened to providing a model with a tailored textbook for information retrieval?',\n   'RAG')],\n '5e9753d0-3c3e-4ccc-af75-c092e3a85643': [('What are some key issues involved in the retrieval process for RAG?',\n   'Retrieval source, retrieval granularity, pre-processing of the retrieval, and selection of the corresponding embedding model.')],\n '3c82bf8f-27c3-4510-b637-a843c2aba36d': [],\n '57360bbd-cb49-4ee9-b29d-6d03c3ebf87a': [('What is the type of tuning used in the RAG-Robust model?',\n   'Once'),\n  ('What is the dataset-base text chunk tuning method mentioned in the context?',\n   'RAG-Robust')],\n '45452dbb-4e58-42f3-8511-b038a6a55ac5': [('What is the tuning frequency for the ARM-RAG model?',\n   'Iterative'),\n  ('How many times does the model UniMS-RAG perform tuning?', 'Once'),\n  ('What is the tuning frequency for the ARM-RAG model?', 'Iterative')],\n '2d957b14-b001-48bf-b175-8eaf360cde33': [],\n '11e78b50-fed8-4653-830d-45419ccfc786': [('What methodology uses the Prize-Collecting Steiner Tree optimization problem for targeted graph retrieval?',\n   'G-Retriever'),\n  ('What optimization problem does G-Retriever employ for targeted graph retrieval?',\n   'Prize-Collecting Steiner Tree (PCST) optimization problem.'),\n  ('What optimization problem does G-Retriever use for targeted graph retrieval?',\n   'Prize-Collecting Steiner Tree (PCST) optimization problem'),\n  ('What problem does G-Retriever address in relation to LLMs and RAG?',\n   'G-Retriever addresses the limitations of LLMs in understanding and answering questions about textual graphs.')],\n '860ec714-78b2-4f86-9fb0-4797e733af57': [],\n 'd8cba5be-8451-4a2b-b635-c03c48875e6a': [('What is Reverse HyDE?',\n   'Reverse HyDE is a method that involves using LLM to generate questions that can be answered by a document, then calculating the similarity between the original question and the hypothetical question during retrieval to reduce the semantic gap between the question and the answer.'),\n  ('What is Reverse HyDE?',\n   'Reverse HyDE is a method that involves using LLM to generate questions that can be answered by a document, then calculating the similarity between the original question and the hypothetical question during retrieval to reduce the semantic gap between the question and the answer.'),\n  ('What is Reverse HyDE?',\n   'Reverse HyDE is a method that involves using LLM to generate questions that can be answered by a document, then calculating the similarity between the original question and the hypothetical question during retrieval to reduce the semantic gap between the question and the answer.')],\n '0da64c4f-025b-41ae-9411-25525d89275e': [],\n '26c4e03c-4907-431d-8f26-7e686852709d': [],\n '1c269c50-ad19-4044-9390-36f004df0cf6': [],\n '5e3b7242-c809-4aa2-ac1f-3dee48f12c35': [],\n '04e735e0-ae16-47e4-bd53-830c9d0b3529': [('What is reranking in the RAG system?',\n   'Reranking fundamentally reorders document chunks to highlight the most pertinent results first, effectively reducing the overall document pool.'),\n  ('What is one method for adjusting the retrieved content in the RAG system?',\n   'Reranking')],\n '2859d9ba-7d65-4802-a99d-1c757e4a2c76': [('What approach does PRCA use to tackle the issue of prompt compression?',\n   'PRCA tackles the issue by training an information extractor.'),\n  ('What approach does PRCA use to tackle the issue of prompt compression?',\n   'PRCA tackles the issue by training an information extractor.')],\n 'fe512429-fb44-4275-983f-03192ede41f9': [('What is the potential approach for aligning LLM outputs with human or retriever preferences?',\n   'Manually annotating the final generated answers and providing feedback through reinforcement learning.'),\n  ('What is a potential approach to align LLM outputs with human or retriever preferences?',\n   'Manually annotating the final generated answers and providing feedback through reinforcement learning.')],\n 'a17f920e-5820-4301-9ab1-596928dcde5e': [],\n 'f8ecce2e-1394-45e7-8839-397c58f51063': [('What is multi-hop retrieval designed to delve deeper into?',\n   'Multi-hop retrieval is designed to delve deeper into graph-structured data sources, extracting interconnected information.'),\n  ('What is multi-hop retrieval designed to delve deeper into?',\n   'Graph-structured data sources.')],\n 'af5f3776-736b-48df-8408-d4a4bb2d247c': [('What is the core task of RAG models?',\n   'The core task of RAG models is Question Answering (QA).'),\n  ('What is the core task of RAG models?', 'Question Answering (QA)'),\n  ('What is the core task of RAG models?',\n   'The core task of RAG models is Question Answering (QA).')],\n '654fabe3-fda0-4385-a7b1-867622175ee5': [('What are some common metrics used to evaluate answer quality in question answering tasks?',\n   'BLEU and ROUGE metrics are commonly used to evaluate answer quality.')],\n 'addee3ab-bd26-4f79-9122-8c33a6b94609': [('What does Noise Robustness appraise in RAG evaluation?',\n   \"The model's capability to manage noise documents that are question-related but lack substantive information.\"),\n  ('What does Noise Robustness appraise in the RAG evaluation?',\n   \"The model's capability to manage noise documents that are question-related but lack substantive information.\")],\n '9cd9f849-ed29-47c5-9a9d-877740e28875': [('What dataset is associated with the task of Single-hop Natural Question in RAG?',\n   'NQ (Natural Question)')],\n '7f762ce2-dbc3-4949-9e16-893b7ef13c70': [('What dataset is associated with Dialog Generation Wizard of Wikipedia (WoW)?',\n   'KBP')],\n '6a421de2-8037-41d5-a782-68de6defb3c0': [('What are some of the metrics applicable for evaluating aspects of RAG models?',\n   'Accuracy, EM, Recall, Precision, R-Rate, Cosine Similarity, Hit Rate, MRR, NDCG, BLEU, ROUGE/ROUGE-L'),\n  ('What are some of the metrics used for evaluating the aspects of RAG models?',\n   'Accuracy, EM, Recall, Precision, R-Rate, Cosine Similarity, Hit Rate, MRR, NDCG, BLEU, ROUGE/ROUGE-L')],\n '780aa918-07ef-4f6a-8baa-15e2c30ffde1': [],\n 'c750d3db-2335-4664-9b80-b40d515cc058': [],\n 'e8b27da0-e92a-43cb-881c-79e3ccb8458c': [('What are the quantitative metrics used for evaluating the TruLens evaluation framework?',\n   'BLEU, ROUGE-L, BertScore, RAGQuestEval')],\n 'd031556e-9891-4059-8b98-c7a23f3dea58': [],\n '41454dc7-0a7f-43bf-91ab-30ffff3927e0': [('What method retrieves and stitches together audio clips to convert machine-translated data into speech-translated data?',\n   'The GSS method.'),\n  ('What method retrieves and stitches together audio clips to convert machine-translated data into speech-translated data?',\n   'The GSS method.')],\n 'd0bc297c-5968-4d35-931d-656f1f181fa0': [],\n 'd1c50b3b-ce1c-4b87-9ee4-306f3c6368a0': [('What is the title of the paper by D. Arora, A. Kini, S. R. Chowdhury, N. Natarajan, G. Sinha, and A. Sharma?',\n   'Gar-meets-rag paradigm for zero-shot information retrieval'),\n  ('What is the title of the paper by D. Arora, A. Kini, S. R. Chowdhury, N. Natarajan, G. Sinha, and A. Sharma?',\n   '\"Gar-meets-rag paradigm for zero-shot information retrieval\"'),\n  ('What is the title of the paper by D. Arora, A. Kini, S. R. Chowdhury, N. Natarajan, G. Sinha, and A. Sharma?',\n   '\"Gar-meets-rag paradigm for zero-shot information retrieval\"'),\n  ('What is the title of the paper by D. Arora, A. Kini, S. R. Chowdhury, N. Natarajan, G. Sinha, and A. Sharma?',\n   '\"Gar-meets-rag paradigm for zero-shot information retrieval\"')],\n '2b8ea0aa-a810-4a50-8a53-2eb197dfea8c': [],\n 'e8b31fee-ef84-462c-9169-9b29eeacf4d2': [('What is the title of the article by A. H. Raudaschl?',\n   'Forget rag, the future is rag-fusion.'),\n  ('What is the title of the article by A. H. Raudaschl mentioned in the context?',\n   '\"Forget rag, the future is rag-fusion\"'),\n  ('What is the title of the article by A. H. Raudaschl discussing the future of RAG models?',\n   'Forget rag, the future is rag-fusion')],\n 'd949cc5a-5983-4f18-aaf4-31413d2ce606': [('What is the title of the paper authored by Z. Sun, X. Wang, Y. Tay, Y. Yang, and D. Zhou?',\n   'Recitation-augmented language models')],\n 'e3d407f9-62f8-46b4-847a-6ad2819fa7b7': [('What is the title of the paper with the arXiv preprint number arXiv:2310.11511?',\n   'Self-rag: Learning to retrieve, generate, and critique through self-reflection')],\n 'd626cc22-1a5e-4827-870c-d21b288205db': [('What is the title of the paper authored by M. Seo, J. Baek, J. Thorne, and S. J. Hwang?',\n   '\"Retrieval-augmented data augmentation for low-resource domain tasks\"'),\n  ('What is the title of the paper by M. Seo, J. Baek, J. Thorne, and S. J. Hwang published in 2024?',\n   'Retrieval-augmented data augmentation for low-resource domain tasks')],\n 'eb7c3907-0088-4dfc-90a7-f82264f9766f': [('What is the title of the paper by G. Izacard, P. Lewis, M. Lomeli, L. Hosseini, F. Petroni, T. Schick, J. Dwivedi-Yu, A. Joulin, S. Riedel, and E. Grave?',\n   'Few-shot learning with retrieval augmented language models')],\n 'e9b63b4a-d39f-444a-b49a-b5d3e6633370': [('What is the title of the paper by S. Siriwardhana, R. Weerasekera, E. Wen, T. Kaluarachchi, R. Rana, and S. Nanayakkara?',\n   '\"Improving the domain adaptation of retrieval augmented generation (rag) models for open domain question answering\"'),\n  ('What is the title of the paper by Z. Yu, C. Xiong, S. Yu, and Z. Liu?',\n   '\"Augmentation-adapted retriever improves generalization of language models as generic plug-in\"'),\n  ('Who authored the paper \"Improving the domain adaptation of retrieval augmented generation (rag) models for open domain question answering\"?',\n   'S. Siriwardhana, R. Weerasekera, E. Wen, T. Kaluarachchi, R. Rana, and S. Nanayakkara')],\n '72faa3f5-f64c-4e30-97c4-b581d3d55a45': [('What is the title of the paper by F. Cuconasu, G. Trappolini, F. Siciliano, S. Filice, C. Campagnano, Y. Maarek, N. Tonellotto, and F. Silvestri?',\n   'The power of noise: Redefining retrieval for rag systems'),\n  ('What is the title of the paper by F. Cuconasu, G. Trappolini, F. Siciliano, S. Filice, C. Campagnano, Y. Maarek, N. Tonellotto, and F. Silvestri?',\n   'The power of noise: Redefining retrieval for rag systems')],\n 'e6e14b75-512f-4e6a-998d-9aa2b8a213e2': [],\n 'bf634d09-310f-4516-bbc3-dac59de46c3a': [('What is the title of the paper by R. Ren et al. in 2023?',\n   '\"Investigating the factual knowledge boundary of large language models with retrieval augmentation\"'),\n  ('What is the title of the paper by R. Ren, Y. Wang, Y. Qu, W. X. Zhao, J. Liu, H. Tian, H. Wu, J.-R. Wen, and H. Wang?',\n   'Investigating the factual knowledge boundary of large language models with retrieval augmentation')],\n '20ec5847-874a-47cf-bf61-f3136b83760e': [('What is the title of the paper authored by S. Zhuang, B. Liu, B. Koopman, and G. Zuccon?',\n   '\"Open-source large language models are strong zero-shot query likelihood models for document ranking\"'),\n  ('What is the title of the paper with arXiv preprint number arXiv:2310.13243?',\n   'Open-source large language models are strong zero-shot query likelihood models for document ranking')],\n '9f1c174f-5257-4320-881e-613aacfe64a7': [('What is the title of the paper by X. Jiang, R. Zhang, Y. Xu, R. Qiu, and Y.?',\n   'Fabula: Intelligence report generation using retrieval-augmented narrative construction')],\n '020a958f-971e-4fc9-9a4c-478994f0731d': [('What is the title of the paper authored by X. He, Y. Tian, Y. Sun, N. V. Chawla, T. Laurent, Y. LeCun, X. Bresson, and B. Hooi?',\n   'G-retriever: Retrieval-augmented generation for textual graph understanding and question answering'),\n  ('What is the title of the paper by X. He, Y. Tian, Y. Sun, N. V. Chawla, T. Laurent, Y. LeCun, X. Bresson, and B. Hooi?',\n   'G-retriever: Retrieval-augmented generation for textual graph understanding and question answering')],\n 'ef1d93b8-c357-4900-bf23-7c7942adca62': [],\n 'db847af5-3247-45ee-b4f5-feda06241715': [('What is the title of the document available at https://python.langchain.com/docs/modules/data connection/document transformers/recursive text splitter?',\n   'Recursively split by character'),\n  ('What is the title of the document by Langchain on recursively splitting text by character?',\n   '\"Recursively split by character\"')],\n '849c00af-457f-4aac-8c02-caad1a279ce9': [],\n '21697402-eba8-4aac-a09b-3da4b08b2770': [('What is the title of the paper by X. Li, R. Zhao, Y. K. Chia, B. Ding, L. Bing, S. Joty, and S. Poria?',\n   'Chain of knowledge: A framework for grounding large language models with structured knowledge bases'),\n  ('What is the title of the paper authored by X. Li, R. Zhao, Y. K. Chia, B. Ding, L. Bing, S. Joty, and S. Poria in 2023?',\n   'Chain of knowledge: A framework for grounding large language models with structured knowledge bases')],\n '727cb00f-c869-48c0-bdf2-9eeb029e69a9': [('What is the title of the paper by T. Kwiatkowski et al. published in 2019?',\n   'Natural questions: a benchmark for question answering research'),\n  ('What is the title of the paper by Y. Liu, S. Yavuz, R. Meng, M. Moorthy, S. Joty, C. Xiong, and Y. Zhou?',\n   '\"Exploring the integration strategies of retriever and large language models\"')],\n '5f617525-d869-4a28-a5b4-e1481a1c2a36': [('What is the title of the paper by Trivedi et al. published in 2022?',\n   'Musique: Multihop questions via single-hop question composition')],\n 'c10a53ac-139b-40a2-8b2d-f2e2a26289d6': [('What is the title of the paper authored by I. Stelmakh, Y. Luan, B. Dhingra, and M.-W. Chang in 2022?',\n   'ASQA: Factoid questions meet long-form answers'),\n  ('What is the title of the paper authored by I. Stelmakh, Y. Luan, B. Dhingra, and M.-W. Chang in 2022?',\n   'ASQA: Factoid questions meet long-form answers'),\n  ('What is the title of the paper authored by I. Stelmakh, Y. Luan, B. Dhingra, and M.-W. Chang in 2022?',\n   'ASQA: Factoid questions meet long-form answers'),\n  ('What is the title of the paper authored by I. Stelmakh, Y. Luan, B. Dhingra, and M.-W. Chang in 2022?',\n   'ASQA: Factoid questions meet long-form answers')],\n '78833254-2ce1-41f9-a1c1-fc5f56a06921': [],\n '8c5414cd-e0e5-46d0-9b54-dbd7201cc273': [('What is the title of the paper with arXiv preprint number arXiv:2104.05919?',\n   'Document-level event argument extraction by conditional generation'),\n  ('What is the title of the paper with arXiv preprint number arXiv:1911.03766?',\n   'Multi-sentence argument linking'),\n  ('What is the title of the paper with arXiv preprint number arXiv:2203.05797?',\n   '\"Long time no see! open-domain conversation with long-term persona memory\"')],\n '67d501ac-d53c-4b55-aaf4-3b85ae0834db': [('When was the paper titled \"The cot collection: Improving zero-shot and few-shot learning of language models via chain-of-thought fine-tuning\" published?',\n   '2023'),\n  ('When was the paper \"The cot collection: Improving zero-shot and few-shot learning of language models via chain-of-thought fine-tuning\" published?',\n   '2023'),\n  ('When was the paper \"The cot collection: Improving zero-shot and few-shot learning of language models via chain-of-thought fine-tuning\" published?',\n   '2023')],\n 'd0126fef-241f-4d48-827b-6ccedf8aba39': [],\n '96c7034e-1206-4ae7-b576-057adf794511': [('When was the paper \"Training verifiers to solve math word problems\" published?',\n   '2021.')],\n 'b66d4b86-e98f-41f2-ae85-c3f875b545db': [('What is the title of the article by C. Jarvis and J. Allard about maximizing LLM performance?',\n   'A survey of techniques for maximizing LLM performance'),\n  ('What is the title of the article by J. Chen, H. Lin, X. Han, and L. Sun?',\n   'Benchmarking large language models in retrieval-augmented generation')],\n 'c0c9ebd2-5c65-4ae2-b2ef-0b3e4337abdb': [],\n '5f43a58b-435c-43aa-84de-d22a033023a2': [('Who are the authors of the paper titled \"Visualize before you write: Imagination-guided open-ended text generation\"?',\n   'W. Zhu, A. Yan, Y. Lu, W. Xu, X. E. Wang, M. Eckstein, and W. Y. Wang')],\n '5d0a414f-066c-473a-9c6e-cb079ba62340': [],\n 'a4ccd98d-83b0-4978-b851-102cd0259bf9': [('What is the proposed analysis framework for benchmarks of RAG systems?',\n   'RGAR (Retrieval, Generation, Additional Requirement)'),\n  ('What is the proposed analysis framework for benchmarks of RAG systems?',\n   'RGAR (Retrieval, Generation, Additional Requirement)')],\n '95370957-6751-4745-87ee-226b41e49a99': [('What is a critical challenge faced by standalone generative models?',\n   'The tendency to produce responses that may not be grounded in facts.')],\n 'e333760d-bdfd-4ba4-b9ab-c824244350df': [('What are the two primary components of the RAG system?',\n   'Retrieval and Generation'),\n  ('What are the two primary components of the RAG system?',\n   'Retrieval and Generation'),\n  ('What are the two primary components of the RAG system?',\n   'Retrieval and Generation')],\n '2621e81a-9764-4af1-a1bf-6ff0fc6d25be': [],\n 'ed43495a-455a-4dc3-8e18-4b726cf2d2f6': [('What is the main focus of the survey on RAG evaluation?',\n   'The main focus of the survey on RAG evaluation is to address the current gaps in the area and provide a comprehensive understanding of the RAG evaluation.'),\n  ('What are the three parts involved in evaluating RAG systems according to the context?',\n   'retrieval, generation, and the whole system')],\n '035c4c29-6cc4-4c0b-b28d-f86cc41ebf7d': [('What are some key challenges in evaluating the retrieval component of RAG systems?',\n   'The dynamic and vast nature of potential knowledge bases, as well as the temporal aspect of information.')],\n 'ad1a8852-8d66-439a-a78c-43b64da99113': [],\n '25254c67-4afc-4cfd-b8b0-b51019c07fef': [('What is the analysis framework proposed for RAG benchmarks?',\n   'RGAR (Retrieval, Generation, and Additional Requirement)')],\n '3ad02593-3465-46fb-8029-57ed3d0c9bbe': [('What does the Relevance evaluation in the RAG system measure?',\n   'It measures the precision and specificity of the retrieval process.'),\n  ('What does the Accuracy component in the evaluation of the retrieval component assess?',\n   'Accuracy assesses how accurate the retrieved documents are in comparison to a set of candidate documents.'),\n  ('What does the Relevance evaluation in the RAG system measure?',\n   'It measures the precision and specificity of the retrieval process.')],\n 'b04de050-6775-402b-a3e8-884ee4a298a5': [('What does the Relevance measure in the context of response and query alignment?',\n   'It measures how well the generated response aligns with the intent and content of the initial query.'),\n  ('What does Relevance (Response ↔Query) measure in the context provided?',\n   'How well the generated response aligns with the intent and content of the initial query.')],\n 'ca1da10e-2f5f-4462-b4d0-a925f3120ee9': [('What is the metric used to evaluate the Accuracy of Retrieval in the RAG systems?',\n   'BLEU, ROUGE-L'),\n  ('What is the metric used to evaluate the Accuracy of the RAG system in the FeB4RAG benchmark?',\n   'Accuracy')],\n 'e6c4d397-137e-4223-831d-945e3b5ab8ba': [],\n '75979f60-8b1b-4903-9c19-1166be16977d': [('What does the Noise Robustness requirement assess in RAG systems?',\n   'Noise Robustness assesses how well the system handles irrelevant information without affecting response quality.')],\n '26a88ffe-cc88-42a2-b6ac-75111e5d14e3': [('What does CRUD-RAG benchmark evaluate systems across?',\n   'CRUD-RAG benchmark evaluates systems across diverse tasks, including text continuation, question answering, hallucination modification, and multi-document summarization.'),\n  ('What is the benchmark dataset used for the CRUD-RAG evaluation?',\n   'Generated (Source: News)')],\n '1db99530-e599-449e-9406-b538e7f2b3c9': [(\"Which benchmarks have built their own datasets using online news articles to test RAG systems' ability to handle real-world information?\",\n   'RGB, MultiHop-RAG, CRUD-RAG, and CDQA.')],\n 'd26359eb-0e2d-49f3-8f7c-b4dcaf0cb88f': [],\n '8e1f8f3c-fd07-4a9f-92c0-9463d6c053ab': [],\n '62dbf956-ce44-4281-9eee-d9ced0c5a5ff': [('What is the formula for Mean Reciprocal Rank (MRR)?',\n   'MRR = 1 / |Q| * Σ(i=1 to |Q|) 1 / rank i')],\n 'ee28f952-b60f-463f-b254-a9359e0b50d2': [('What are some metrics used to evaluate RAG systems?',\n   'Misleading Rate, Mistake Reappearance Rate, and Error Detection Rate'),\n  ('What are some metrics used to evaluate RAG systems?',\n   'Misleading Rate, Mistake Reappearance Rate, and Error Detection Rate.')],\n 'fb1b2181-fe61-431f-8d4e-194ac7604c33': [('What is ROUGE used to evaluate?',\n   'ROUGE is used to evaluate the quality of summaries by comparing them to human-generated reference summaries.')],\n '613275f1-abbc-4e83-b491-b6b7363c071e': [('What criteria are used by LLMs as judges to score generated text?',\n   'coherence, relevance, and fluency.')],\n 'd59b3c68-afe3-4d4c-ada6-79a9bd624163': [('What does the metric \"Single Query Latency\" measure in RAG systems?',\n   'The average time taken to process a single query, including both retrieval and generating phases.'),\n  ('What does the Rejection Rate metric measure in RAG systems?',\n   'The rate at which the system refrains from generating a response.')],\n 'f97e6c8c-5f11-4ebc-a0f7-5b3e548213fa': [],\n 'e0102430-a370-4488-96c9-ed2495cbb343': [('What is a challenge when using LLMs as judges for chatbot responses?',\n   'Challenges in aligning with human judgment, establishing effective grading scales, and applying consistent evaluation across varied use cases.'),\n  ('What are recent strategies that offer novel angles for evaluating RAG systems?',\n   'CRUD-based assessments'),\n  ('What are recent strategies, such as CRUD-based assessments, scrutinizing in RAG systems?',\n   'The interactive capabilities with dynamic information environments.')],\n '5ab9fbad-ef02-4901-86a5-3d707c7498dc': [],\n '4f17104e-8468-4705-86e2-2ea4aea56e24': [('Who are the authors of the survey mentioned in the context?',\n   'Hao Yu, Aoran Gan, Kai Zhang, Shiwei Tong, Qi Liu, and Zhaofeng Liu.'),\n  ('Who conducted the survey mentioned in the context?',\n   'Hao Yu, Aoran Gan, Kai Zhang, Shiwei Tong, Qi Liu, and Zhaofeng Liu.')],\n '9a8f64bb-b586-4be8-9275-451a97ed83c5': [('What is the title of the tech report by Balaguer et al. comparing RAG vs Fine-tuning?',\n   'RAG vs Fine-tuning: Pipelines, Tradeoffs, and a Case Study on Agriculture.'),\n  ('What is the title of the tech. rep. by Balaguer et al. comparing RAG vs Fine-tuning?',\n   'RAG vs Fine-tuning: Pipelines, Tradeoffs, and a Case Study on Agriculture.')],\n 'fdcda531-8e61-443b-9ed3-5856429e7ecd': [],\n '5d798b3c-c405-48d9-8fa0-e36ae88265ab': [('What is the title of the tech report by Gao et al. in 2024?',\n   'Retrieval-Augmented Generation for Large Language Models: A Survey')],\n 'fada6d70-bcbc-4a95-98f6-82d3f5d9780b': [],\n 'dd76a94c-7916-4558-8705-e55d86282837': [],\n '382ddcf1-7f38-4276-b8ce-40305e09603f': [('When was the paper \"Best Practices for LLM Evaluation of RAG Applications\" published?',\n   'December 2023'),\n  ('When was the paper \"Best Practices for LLM Evaluation of RAG Applications\" published?',\n   'December 2023'),\n  ('When was the paper \"Best Practices for LLM Evaluation of RAG Applications\" published?',\n   'December 2023')],\n '27d410c8-303e-49e2-859a-f2b8bd6d690d': [('What is the title of the tech. rep. by Lewis et al. mentioned in the context?',\n   'Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks.'),\n  ('What is the title of the tech report authored by Lewis et al. in April 2021?',\n   'Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks'),\n  ('When was the benchmark Uhgeval for the hallucination of Chinese large language models published?',\n   'Uhgeval benchmark was published in 2023.')],\n '21865889-7eb5-4329-8f81-e4ce9c4cd673': [('What is the title of the benchmark for knowledge intensive language tasks discussed in the context?',\n   'KILT')],\n '6fb7b7dc-0343-4b74-a933-116ef7482352': [('What is the title of the paper by Thorne, J., Vlachos, A., Christodoulopoulos, C., Mittal?',\n   'FEVER: a large-scale dataset for fact extraction and VERification.')],\n '2b4c2cd0-5e4c-4e6b-bb2f-8d1d69b4ae42': [('What is the title of the paper authored by Hao Yu, Aoran Gan, Kai Zhang, Shiwei Tong, Qi Liu, and Zhaofeng Liu?',\n   'Chain-of-thought prompting elicits reasoning in large language models'),\n  ('What is the title of the paper by Wang, A., Pruksachatkun, Y., Nangia, N., Singh, A., Michael, J., Hill, F., Levy, O., and Bowman, S.R.?',\n   'SuperGLUE: A stickier benchmark for general-purpose language understanding systems')],\n '5a68c7c7-d936-4806-8fcc-c7bf3810471b': [],\n '5b2ee39b-b0b6-46f1-bb17-b487af02cbb1': [('When was the paper \"Judging llm-as-a-judge with mt-bench and chatbot arena\" published?',\n   'June 2023'),\n  ('When was the paper \"Retrieving and Reading: A Comprehensive Survey on Open-domain Question Answering\" published?',\n   'May 2021'),\n  ('When was the paper \"Retrieving and Reading: A Comprehensive Survey on Open-domain Question Answering\" published?',\n   'May 2021')],\n '85590f31-e883-433f-a811-05775eb956fb': [('What are the three types of retrieval components in RAG systems?',\n   'sparse retrieval, dense retrieval, web search engine'),\n  ('What are the three types of retrieval components in RAG systems?',\n   'sparse retrieval, dense retrieval, web search engine'),\n  ('What are the three types of retrieval components in RAG systems?',\n   'sparse retrieval, dense retrieval, and web search engine')],\n '70e7effe-9404-440d-86f0-324822299c08': [],\n '84645a32-3643-4628-b4bd-201e3902fdca': [],\n 'bf9996d1-6751-47e2-9b40-36e8f80f2cf5': []}"
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc_id_2_gen_questions"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "from datetime import datetime\n",
    "\n",
    "\n",
    "def get_current_timestamp():\n",
    "    now = datetime.now()\n",
    "    return str(now).split(\".\")[0].replace(\" \", \"-\").replace(\":\", \"-\")\n",
    "\n",
    "cur_dir_str = f\"evaluations/{get_current_timestamp()}\"\n",
    "cur_dir = Path(cur_dir_str)\n",
    "cur_dir.mkdir(parents=True, exist_ok=True)\n",
    "\n",
    "pd.DataFrame(outputs).to_csv(f\"{cur_dir_str}/intermediate_output.csv\", index=False)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Setup critique agents"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "outputs": [],
   "source": [
    "question_groundedness_critique_prompt = \"\"\"\n",
    "You will be given a context and a question.\n",
    "Your task is to provide a 'total rating' scoring how well one can answer the given question unambiguously with the given context.\n",
    "Give your answer on a scale of 1 to 5, where 1 means that the question is not answerable at all given the context, and 5 means that the question is clearly and unambiguously answerable with the context.\n",
    "\n",
    "Provide your answer as follows:\n",
    "\n",
    "Answer:::\n",
    "Evaluation: (your rationale for the rating, as a text)\n",
    "Total rating: (your rating, as a number between 1 and 5)\n",
    "\n",
    "You MUST provide values for 'Evaluation:' and 'Total rating:' in your answer.\n",
    "\n",
    "Now here are the question and context.\n",
    "\n",
    "Question: {question}\\n\n",
    "Context: {context}\\n\n",
    "Answer::: \"\"\"\n",
    "\n",
    "question_relevance_critique_prompt = \"\"\"\n",
    "You will be given a question.\n",
    "Your task is to provide a 'total rating' representing how useful this question can be to machine learning developers building Retrieval-augmented generation(RAG) application with the RAG survey paper.\n",
    "Give your answer on a scale of 1 to 5, where 1 means that the question is not useful at all, and 5 means that the question is extremely useful.\n",
    "\n",
    "Provide your answer as follows:\n",
    "\n",
    "Answer:::\n",
    "Evaluation: (your rationale for the rating, as a text)\n",
    "Total rating: (your rating, as a number between 1 and 5)\n",
    "\n",
    "You MUST provide values for 'Evaluation:' and 'Total rating:' in your answer.\n",
    "\n",
    "Now here is the question.\n",
    "\n",
    "Question: {question}\\n\n",
    "Answer::: \"\"\"\n",
    "\n",
    "question_standalone_critique_prompt = \"\"\"\n",
    "You will be given a question.\n",
    "Your task is to provide a 'total rating' representing how context-independant this question is.\n",
    "Give your answer on a scale of 1 to 5, where 1 means that the question depends on additional information to be understood, and 5 means that the question makes sense by itself.\n",
    "For instance, if the question refers to a particular setting, like 'in the context' or 'in the document', the rating must be 1.\n",
    "The questions can contain obscure technical nouns or acronyms like Gradio, Hub, Hugging Face or Space and still be a 5: it must simply be clear to an operator with access to documentation what the question is about.\n",
    "\n",
    "For instance, \"What is the name of the checkpoint from which the ViT model is imported?\" should receive a 1, since there is an implicit mention of a context, thus the question is not independant from the context.\n",
    "\n",
    "Provide your answer as follows:\n",
    "\n",
    "Answer:::\n",
    "Evaluation: (your rationale for the rating, as a text)\n",
    "Total rating: (your rating, as a number between 1 and 5)\n",
    "\n",
    "You MUST provide values for 'Evaluation:' and 'Total rating:' in your answer.\n",
    "\n",
    "Now here is the question.\n",
    "\n",
    "Question: {question}\\n\n",
    "Answer::: \"\"\""
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 150/150 [09:16<00:00,  3.71s/it]\n"
     ]
    }
   ],
   "source": [
    "for output in tqdm(outputs):\n",
    "    evaluations = {\n",
    "        \"groundedness\": call_llm(question_groundedness_critique_prompt.format(\n",
    "            context=output[\"context\"], question=output[\"question\"]\n",
    "        )),\n",
    "        \"relevance\": call_llm(question_relevance_critique_prompt.format(\n",
    "            question=output[\"question\"]\n",
    "        )),\n",
    "        \"standalone\": call_llm(question_standalone_critique_prompt.format(\n",
    "            question=output[\"question\"]\n",
    "        ))\n",
    "    }\n",
    "\n",
    "    try:\n",
    "        for criterion, evaluation in evaluations.items():\n",
    "            score, eval = (\n",
    "                int(evaluation.split(\"Total rating: \")[-1].strip()),\n",
    "                evaluation.split(\"Total rating: \")[-2].split(\"Evaluation: \")[1],\n",
    "            )\n",
    "            output.update(\n",
    "                {\n",
    "                    f\"{criterion}_score\": score,\n",
    "                    f\"{criterion}_eval\": eval\n",
    "                }\n",
    "            )\n",
    "    except Exception as e:\n",
    "        continue"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation dataset before filtering:\n"
     ]
    },
    {
     "data": {
      "text/plain": "                                                                                                                                              question  \\\n0                                                                                                                 What is reranking in the RAG system?   \n1                                                                                              What is the main focus of the survey on RAG evaluation?   \n2                                                                                                                 What is the core task of RAG models?   \n3                                               What is the title of the paper by X. Li, R. Zhao, Y. K. Chia, B. Ding, L. Bing, S. Joty, and S. Poria?   \n4                                                             What is the title of the paper authored by S. Zhuang, B. Liu, B. Koopman, and G. Zuccon?   \n..                                                                                                                                                 ...   \n145  When was the paper \"The cot collection: Improving zero-shot and few-shot learning of language models via chain-of-thought fine-tuning\" published?   \n146                                                              When was the paper \"Best Practices for LLM Evaluation of RAG Applications\" published?   \n147                                                                                        What is the analysis framework proposed for RAG benchmarks?   \n148                                                                    What is the title of the paper with the arXiv preprint number arXiv:2310.11511?   \n149                                                           What are recent strategies, such as CRUD-based assessments, scrutinizing in RAG systems?   \n\n                                                                                                                                                         answer  \\\n0               Reranking fundamentally reorders document chunks to highlight the most pertinent results first, effectively reducing the overall document pool.   \n1    The main focus of the survey on RAG evaluation is to address the current gaps in the area and provide a comprehensive understanding of the RAG evaluation.   \n2                                                                                                       The core task of RAG models is Question Answering (QA).   \n3                                                           Chain of knowledge: A framework for grounding large language models with structured knowledge bases   \n4                                                         \"Open-source large language models are strong zero-shot query likelihood models for document ranking\"   \n..                                                                                                                                                          ...   \n145                                                                                                                                                        2023   \n146                                                                                                                                               December 2023   \n147                                                                                                    RGAR (Retrieval, Generation, and Additional Requirement)   \n148                                                                              Self-rag: Learning to retrieve, generate, and critique through self-reflection   \n149                                                                                         The interactive capabilities with dynamic information environments.   \n\n     groundedness_score  relevance_score  standalone_score  \n0                   5.0              NaN               NaN  \n1                   5.0              4.0               5.0  \n2                   5.0              5.0               5.0  \n3                   5.0              1.0               5.0  \n4                   NaN              NaN               NaN  \n..                  ...              ...               ...  \n145                 5.0              4.0               5.0  \n146                 5.0              4.0               5.0  \n147                 5.0              4.0               5.0  \n148                 5.0              3.0               5.0  \n149                 3.0              5.0               1.0  \n\n[150 rows x 5 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>question</th>\n      <th>answer</th>\n      <th>groundedness_score</th>\n      <th>relevance_score</th>\n      <th>standalone_score</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>What is reranking in the RAG system?</td>\n      <td>Reranking fundamentally reorders document chunks to highlight the most pertinent results first, effectively reducing the overall document pool.</td>\n      <td>5.0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>What is the main focus of the survey on RAG evaluation?</td>\n      <td>The main focus of the survey on RAG evaluation is to address the current gaps in the area and provide a comprehensive understanding of the RAG evaluation.</td>\n      <td>5.0</td>\n      <td>4.0</td>\n      <td>5.0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>What is the core task of RAG models?</td>\n      <td>The core task of RAG models is Question Answering (QA).</td>\n      <td>5.0</td>\n      <td>5.0</td>\n      <td>5.0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>What is the title of the paper by X. Li, R. Zhao, Y. K. Chia, B. Ding, L. Bing, S. Joty, and S. Poria?</td>\n      <td>Chain of knowledge: A framework for grounding large language models with structured knowledge bases</td>\n      <td>5.0</td>\n      <td>1.0</td>\n      <td>5.0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>What is the title of the paper authored by S. Zhuang, B. Liu, B. Koopman, and G. Zuccon?</td>\n      <td>\"Open-source large language models are strong zero-shot query likelihood models for document ranking\"</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>145</th>\n      <td>When was the paper \"The cot collection: Improving zero-shot and few-shot learning of language models via chain-of-thought fine-tuning\" published?</td>\n      <td>2023</td>\n      <td>5.0</td>\n      <td>4.0</td>\n      <td>5.0</td>\n    </tr>\n    <tr>\n      <th>146</th>\n      <td>When was the paper \"Best Practices for LLM Evaluation of RAG Applications\" published?</td>\n      <td>December 2023</td>\n      <td>5.0</td>\n      <td>4.0</td>\n      <td>5.0</td>\n    </tr>\n    <tr>\n      <th>147</th>\n      <td>What is the analysis framework proposed for RAG benchmarks?</td>\n      <td>RGAR (Retrieval, Generation, and Additional Requirement)</td>\n      <td>5.0</td>\n      <td>4.0</td>\n      <td>5.0</td>\n    </tr>\n    <tr>\n      <th>148</th>\n      <td>What is the title of the paper with the arXiv preprint number arXiv:2310.11511?</td>\n      <td>Self-rag: Learning to retrieve, generate, and critique through self-reflection</td>\n      <td>5.0</td>\n      <td>3.0</td>\n      <td>5.0</td>\n    </tr>\n    <tr>\n      <th>149</th>\n      <td>What are recent strategies, such as CRUD-based assessments, scrutinizing in RAG systems?</td>\n      <td>The interactive capabilities with dynamic information environments.</td>\n      <td>3.0</td>\n      <td>5.0</td>\n      <td>1.0</td>\n    </tr>\n  </tbody>\n</table>\n<p>150 rows × 5 columns</p>\n</div>"
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "============================================\n",
      "Final evaluation dataset:\n"
     ]
    },
    {
     "data": {
      "text/plain": "                                                                                                                                              question  \\\n1                                                                                              What is the main focus of the survey on RAG evaluation?   \n2                                                                                                                 What is the core task of RAG models?   \n5                                                               When was the paper \"Judging llm-as-a-judge with mt-bench and chatbot arena\" published?   \n6                                                                                        What does the Relevance evaluation in the RAG system measure?   \n7                                                                                  What does the metric \"Single Query Latency\" measure in RAG systems?   \n..                                                                                                                                                 ...   \n143                                                                        What are some of the metrics used for evaluating the aspects of RAG models?   \n144        What technology enhances LLMs by retrieving relevant document chunks from external knowledge bases through semantic similarity calculation?   \n145  When was the paper \"The cot collection: Improving zero-shot and few-shot learning of language models via chain-of-thought fine-tuning\" published?   \n146                                                              When was the paper \"Best Practices for LLM Evaluation of RAG Applications\" published?   \n147                                                                                        What is the analysis framework proposed for RAG benchmarks?   \n\n                                                                                                                                                         answer  \\\n1    The main focus of the survey on RAG evaluation is to address the current gaps in the area and provide a comprehensive understanding of the RAG evaluation.   \n2                                                                                                       The core task of RAG models is Question Answering (QA).   \n5                                                                                                                                                     June 2023   \n6                                                                                           It measures the precision and specificity of the retrieval process.   \n7                                                             The average time taken to process a single query, including both retrieval and generating phases.   \n..                                                                                                                                                          ...   \n143                                                        Accuracy, EM, Recall, Precision, R-Rate, Cosine Similarity, Hit Rate, MRR, NDCG, BLEU, ROUGE/ROUGE-L   \n144                                                                                                                        Retrieval-Augmented Generation (RAG)   \n145                                                                                                                                                        2023   \n146                                                                                                                                               December 2023   \n147                                                                                                    RGAR (Retrieval, Generation, and Additional Requirement)   \n\n     groundedness_score  relevance_score  standalone_score  \n1                   5.0              4.0               5.0  \n2                   5.0              5.0               5.0  \n5                   5.0              4.0               5.0  \n6                   5.0              5.0               5.0  \n7                   5.0              4.0               5.0  \n..                  ...              ...               ...  \n143                 5.0              5.0               5.0  \n144                 5.0              5.0               5.0  \n145                 5.0              4.0               5.0  \n146                 5.0              4.0               5.0  \n147                 5.0              4.0               5.0  \n\n[82 rows x 5 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>question</th>\n      <th>answer</th>\n      <th>groundedness_score</th>\n      <th>relevance_score</th>\n      <th>standalone_score</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>1</th>\n      <td>What is the main focus of the survey on RAG evaluation?</td>\n      <td>The main focus of the survey on RAG evaluation is to address the current gaps in the area and provide a comprehensive understanding of the RAG evaluation.</td>\n      <td>5.0</td>\n      <td>4.0</td>\n      <td>5.0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>What is the core task of RAG models?</td>\n      <td>The core task of RAG models is Question Answering (QA).</td>\n      <td>5.0</td>\n      <td>5.0</td>\n      <td>5.0</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>When was the paper \"Judging llm-as-a-judge with mt-bench and chatbot arena\" published?</td>\n      <td>June 2023</td>\n      <td>5.0</td>\n      <td>4.0</td>\n      <td>5.0</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>What does the Relevance evaluation in the RAG system measure?</td>\n      <td>It measures the precision and specificity of the retrieval process.</td>\n      <td>5.0</td>\n      <td>5.0</td>\n      <td>5.0</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>What does the metric \"Single Query Latency\" measure in RAG systems?</td>\n      <td>The average time taken to process a single query, including both retrieval and generating phases.</td>\n      <td>5.0</td>\n      <td>4.0</td>\n      <td>5.0</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>143</th>\n      <td>What are some of the metrics used for evaluating the aspects of RAG models?</td>\n      <td>Accuracy, EM, Recall, Precision, R-Rate, Cosine Similarity, Hit Rate, MRR, NDCG, BLEU, ROUGE/ROUGE-L</td>\n      <td>5.0</td>\n      <td>5.0</td>\n      <td>5.0</td>\n    </tr>\n    <tr>\n      <th>144</th>\n      <td>What technology enhances LLMs by retrieving relevant document chunks from external knowledge bases through semantic similarity calculation?</td>\n      <td>Retrieval-Augmented Generation (RAG)</td>\n      <td>5.0</td>\n      <td>5.0</td>\n      <td>5.0</td>\n    </tr>\n    <tr>\n      <th>145</th>\n      <td>When was the paper \"The cot collection: Improving zero-shot and few-shot learning of language models via chain-of-thought fine-tuning\" published?</td>\n      <td>2023</td>\n      <td>5.0</td>\n      <td>4.0</td>\n      <td>5.0</td>\n    </tr>\n    <tr>\n      <th>146</th>\n      <td>When was the paper \"Best Practices for LLM Evaluation of RAG Applications\" published?</td>\n      <td>December 2023</td>\n      <td>5.0</td>\n      <td>4.0</td>\n      <td>5.0</td>\n    </tr>\n    <tr>\n      <th>147</th>\n      <td>What is the analysis framework proposed for RAG benchmarks?</td>\n      <td>RGAR (Retrieval, Generation, and Additional Requirement)</td>\n      <td>5.0</td>\n      <td>4.0</td>\n      <td>5.0</td>\n    </tr>\n  </tbody>\n</table>\n<p>82 rows × 5 columns</p>\n</div>"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import datasets\n",
    "\n",
    "pd.set_option(\"display.max_colwidth\", None)\n",
    "\n",
    "generated_questions = pd.DataFrame.from_dict(outputs)\n",
    "\n",
    "print(\"Evaluation dataset before filtering:\")\n",
    "display(\n",
    "    generated_questions[\n",
    "        [\n",
    "            \"question\",\n",
    "            \"answer\",\n",
    "            \"groundedness_score\",\n",
    "            \"relevance_score\",\n",
    "            \"standalone_score\",\n",
    "        ]\n",
    "    ]\n",
    ")\n",
    "\n",
    "generated_questions = generated_questions.loc[\n",
    "    (generated_questions[\"groundedness_score\"] >= 4)\n",
    "    & (generated_questions[\"relevance_score\"] >= 4)\n",
    "    & (generated_questions[\"standalone_score\"] >= 4)\n",
    "]\n",
    "print(\"============================================\")\n",
    "print(\"Final evaluation dataset:\")\n",
    "display(\n",
    "    generated_questions[\n",
    "        [\n",
    "            \"question\",\n",
    "            \"answer\",\n",
    "            \"groundedness_score\",\n",
    "            \"relevance_score\",\n",
    "            \"standalone_score\",\n",
    "        ]\n",
    "    ]\n",
    ")"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "eval counts: 82/150\n"
     ]
    }
   ],
   "source": [
    "print(f\"eval counts: {len(generated_questions)}/{len(outputs)}\")"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "outputs": [],
   "source": [
    "generated_questions.to_csv(f\"{cur_dir_str}/eval.csv\", index=False)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "evaluations/2024-05-29-13-14-54\n"
     ]
    }
   ],
   "source": [
    "print(cur_dir_str)"
   ],
   "metadata": {
    "collapsed": false
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}