arxiv_evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import argparse\n",
    "import json\n",
    "import openai\n",
    "from dotenv import load_dotenv\n",
    "import os\n",
    "import yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_params = {\n",
    "    'model': \"gpt-4\",\n",
    "    'temperature': 0,\n",
    "}\n",
    "\n",
    "load_dotenv('../.env')\n",
    "openai.api_key  = os.getenv('OPENAI_API_KEY')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_prompt_template(file):\n",
    "    prompt_dict = {}\n",
    "    current_task = None  # Track the current task, initialized to None\n",
    "\n",
    "    with open(file, 'r') as f:\n",
    "        for line in f:\n",
    "            stripped_line = line.strip()\n",
    "\n",
    "            # Skip comments\n",
    "            if stripped_line.startswith('<!--'):\n",
    "                continue\n",
    "            \n",
    "            if stripped_line.startswith('## '):\n",
    "                # New task starts\n",
    "                current_task = stripped_line[3:]\n",
    "                prompt_dict[current_task] = ''\n",
    "            elif not stripped_line.startswith('# ') and current_task:\n",
    "                # If this line is part of a task (not a task declaration or a section),\n",
    "                # and a task is currently being tracked, append this line to the task.\n",
    "                if not stripped_line.startswith('**Prompt:**'):\n",
    "                    prompt_dict[current_task] += line\n",
    "\n",
    "    return prompt_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "arxiv_number='2111.01152'\n",
    "prompt_template='Prompt_template.md'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt_dict=load_prompt_template(prompt_template)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(f'{arxiv_number}/{arxiv_number}.yaml','r') as f:\n",
    "        kwargs= yaml.safe_load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_prompt(kwargs,prompt_dict):\n",
    "    if 'task' in kwargs:\n",
    "        for key,val in fill_placeholder(kwargs):\n",
    "            if isinstance(val, bool):\n",
    "                content=content.replace(f'[{key}]',f'{key}') if val else content.replace(f'[{key}]','')\n",
    "        content=prompt_dict[kwargs['task']].format(**dict(fill_placeholder(kwargs))).strip()\n",
    "        \n",
    "        message={\n",
    "        'role':'user',\n",
    "        'content':content\n",
    "    }\n",
    "        return message"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "'expression_kinetic'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_9952/2934626881.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmessage\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mgenerate_prompt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprompt_dict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/tmp/ipykernel_9952/3150665171.py\u001b[0m in \u001b[0;36mgenerate_prompt\u001b[0;34m(kwargs, prompt_dict)\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mgenerate_prompt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mprompt_dict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;34m'task'\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m         \u001b[0mcontent\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mprompt_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'task'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mdict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfill_placeholder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstrip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mval\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mfill_placeholder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mval\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbool\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 'expression_kinetic'"
     ]
    }
   ],
   "source": [
    "message=generate_prompt(kwargs[2], prompt_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You will be instructed to describe the kinetic term of Hamiltonian in a system with two layers in the momentum space in the second-quantized form.   \n",
      "The degrees of freedom of the system are: valleys, layers, and momentum.  \n",
      "Express the Kinetic Hamiltonian H_{\\tau} using variables which are only on the diagonal terms, and arrange the basis in the order of bottom layer and top layer. \n",
      "\n",
      "Use the following conventions for the symbols:  \n",
      "$\\tau=\\pm 1$ represents $\\pm K$ valleys, $\\bm{\\kappa}=\\frac{4\\pi}{3a_M}\\left(1,0\\right)$ is at a corner of the moiré Brillouin zone. The $2\\times 2$ Hamiltonian hybridizes the bottom layer ($\\mathfrak{b}$) and top layer ($\\mathfrak{t}$), where the off diagonal terms describe the interlayer tunneling $\\Delta_{\\text{T},\\tau}$, and the diagonal terms describe the momentum-shifted kinetic energy with the effective mass $(m_{\\mathfrak{b}},m_{\\mathfrak{t}})=(0.65,0.35)m_e$ ($m_e$ is the rest electron mass), plus the intralayer potential $\\Delta_{\\mathfrak{b}/\\mathfrak{t}}$, and a band offset $V_{z\\mathfrak{t}}$.\n"
     ]
    }
   ],
   "source": [
    "print(message['content'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fill_placeholder(kwarg):\n",
    "    for key, val in kwarg['placeholder'].items():\n",
    "        if val['human'] is None:\n",
    "            yield key,val['LLM']\n",
    "        else:\n",
    "            yield key,val['LLM']\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<generator object fill_placeholder at 0x7f27d0bc6190>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fill_placeholder()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "'system'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_9952/259903850.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mgenerate_prompt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mprompt_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mprompt_dict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/tmp/ipykernel_9952/2075998152.py\u001b[0m in \u001b[0;36mgenerate_prompt\u001b[0;34m(kwargs, prompt_dict)\u001b[0m\n\u001b[1;32m      5\u001b[0m         \u001b[0;34m'user'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m         \u001b[0;34m'content'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m         \u001b[0mprompt_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'task'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstrip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m     }\n\u001b[1;32m      9\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mmessage\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 'system'"
     ]
    }
   ],
   "source": [
    "generate_prompt(kwargs[1],prompt_dict=prompt_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}