arxiv_score_prompt.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import openai\n",
    "import pandas as pd\n",
    "import json\n",
    "from IPython.display import Markdown\n",
    "import sys\n",
    "from tqdm import tqdm\n",
    "import pyperclip\n",
    "import yaml"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load prompt template"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "sys.path.append('..')\n",
    "from utils import *\n",
    "from functools import partial"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt_dict = partial(load_prompt_template,file='../prompt_template.md')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def drop_text_after(string):\n",
    "    string_new=''\n",
    "    for line in string.split('\\n'):\n",
    "        if '===' in line:\n",
    "            break\n",
    "        else:\n",
    "            if 'You should recall that' not in line:\n",
    "                string_new='\\n'.join([string_new,line])\n",
    "    return string_new[1:]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('arxiv_number.jsonl','r') as f:\n",
    "    kwargs_jsonl= [json.loads(line) for line in f]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('arxiv_number.yaml','r') as f:\n",
    "    kwargs_jsonl= yaml.safe_load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prompt_template(descriptor):\n",
    "    return drop_text_after(prompt_dict()[descriptor['task']])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load automatically filled prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_markdown_to_dict(filename):\n",
    "    with open(filename, 'r') as file:\n",
    "        lines = file.readlines()\n",
    "\n",
    "    result_dict = {}\n",
    "    current_title = None\n",
    "    content_buffer = []\n",
    "\n",
    "    for line in lines:\n",
    "        stripped_line = line.strip()\n",
    "\n",
    "        if stripped_line.startswith(\"# \"):\n",
    "            if current_title:  # If there's already a title detected\n",
    "                result_dict[current_title] = '\\n'.join(content_buffer).strip()\n",
    "                content_buffer = []\n",
    "            current_title = stripped_line[2:]\n",
    "        else:\n",
    "            content_buffer.append(line)\n",
    "\n",
    "    # For the last title-content pair\n",
    "    if current_title and content_buffer:\n",
    "        result_dict[current_title] = '\\n'.join(content_buffer).strip()\n",
    "\n",
    "    return result_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = \"arxiv_number_extractor.md\"\n",
    "filled_dict = parse_markdown_to_dict(filename)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Return place holder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "def extract_filled_values(template_str):\n",
    "    template_str = template_str.replace('{{', '').replace('}}', '')\n",
    "    # Extract placeholders from the template\n",
    "    # placeholders = re.findall(r\"\\{(\\w+)\\}\", template_str)\n",
    "    placeholders = re.findall(r\"[\\{]([\\w\\|\\-$ ,.\\{\\}]+?)[\\}]\", template_str)\n",
    "    placeholders_optional = re.findall(r\"[\\[]([\\w\\|\\-$ ,.\\{\\}]+)[\\]]\", template_str)\n",
    "    # Create a regex pattern to match the filled values based on the placeholders\n",
    "    placeholders_unique=[]\n",
    "    for placeholder in placeholders+placeholders_optional:\n",
    "        if placeholder not in placeholders_unique:\n",
    "            placeholders_unique.append(placeholder)\n",
    "    return placeholders_unique\n",
    "    \n",
    "    # pattern = re.sub(r\"\\{(\\w+)\\}\", r\"(?P<\\1>\\\\S+)\", template_str)\n",
    "\n",
    "    # # Find the values using the generated pattern\n",
    "    # match = re.match(pattern, filled_str)\n",
    "    # if match:\n",
    "    #     return {placeholder: match.group(placeholder) for placeholder in placeholders}\n",
    "    # else:\n",
    "    #     return None\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Energy_dispersion', 'parabolic|Dirac|cos', 'electrons|holes', 'momentum_shift', 'k_symbol', 'shifted_Ek', 'In addition, a shift of {momentum_shift} in the momentum ${k_symbol}$ for {shifted_Ek}, respectively.']\n"
     ]
    }
   ],
   "source": [
    "# Example\n",
    "template_str = drop_text_after(prompt_dict()[\"Define each term in Kinetic Hamiltonian (continuum version)\"])\n",
    "# filled_str = filled_dict['Construct Kinetic Hamiltonian (continuum version, single-particle)']\n",
    "results = extract_filled_values(template_str)\n",
    "print(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "placeholder:\n",
      "    Hartree_Fock_second_quantized_symbol:\n",
      "      LLM: \n",
      "      human: \n",
      "      score: \n",
      "    expected_value:\n",
      "      LLM: \n",
      "      human: \n",
      "      score: \n",
      "    expression_Hartree_1:\n",
      "      LLM: \n",
      "      human: \n",
      "      score: \n",
      "    expression_Hartree_2:\n",
      "      LLM: \n",
      "      human: \n",
      "      score: \n",
      "\n"
     ]
    }
   ],
   "source": [
    "string= 'placeholder:\\n'\n",
    "for result in results:\n",
    "    string+=f'    {result}:\\n'\n",
    "    string+='      LLM: \\n'\n",
    "    string+='      human: \\n'\n",
    "    string+='      score: \\n'\n",
    "    string+='        Haining: \\n'\n",
    "    string+='        Will: \\n'\n",
    "    string+='        Yasaman: \\n'\n",
    "print(string)\n",
    "pyperclip.copy(string)    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Validate yaml"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. Verify score is not None\n",
    "2. Verify if score<2, human is not None; if score == 2, human is None and LLM is not None\n",
    "3. Verify not (LLM and human are both None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [],
   "source": [
    "import yaml\n",
    "with open('arxiv_number.yaml','r') as f:\n",
    "    kwargs_yaml = yaml.safe_load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "string=''\n",
    "for kwargs in kwargs_jsonl:\n",
    "    string+=f\"# {kwargs['task']}\\n\"\n",
    "    string+=drop_text_after(prompt_dict()[kwargs['task']])\n",
    "\n",
    "with open('../cmp2.md','w') as f:\n",
    "    f.write(string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.11"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}