openaiKG.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "57ea0bce-75b1-4e35-a17c-53ae7b1ccff6",
   "metadata": {},
   "source": [
    "# Get Wiki info of OpenAI key stakeholders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2cc4fb2b-08e4-43a0-a37e-6cdd62f5dc83",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install langchain\n",
    "# !pip install wikipedia"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "33a04ef2-16aa-41dc-9ee1-a53703275d41",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the OpenAI Wikipedia page\n",
    "from langchain.document_loaders import WikipediaLoader\n",
    "from langchain.text_splitter import CharacterTextSplitter\n",
    "raw_documents = WikipediaLoader(query=\"OpenAI\").load()\n",
    "\n",
    "# Define chunking strategy\n",
    "text_splitter = CharacterTextSplitter.from_tiktoken_encoder(\n",
    "    chunk_size=1000, chunk_overlap=20\n",
    ")\n",
    "# Chunk the document\n",
    "documents = text_splitter.split_documents(raw_documents)\n",
    "for d in documents:\n",
    "    del d.metadata[\"summary\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "9da3f01d-ae49-4d18-8b27-86932b67a5f3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://en.wikipedia.org/wiki/OpenAI\n",
      "https://en.wikipedia.org/wiki/Removal_of_Sam_Altman_from_OpenAI\n",
      "https://en.wikipedia.org/wiki/OpenAI_Five\n",
      "https://en.wikipedia.org/wiki/Greg_Brockman\n",
      "https://en.wikipedia.org/wiki/Ilya_Sutskever\n",
      "https://en.wikipedia.org/wiki/Emmett_Shear\n",
      "https://en.wikipedia.org/wiki/Artificial_general_intelligence\n",
      "https://en.wikipedia.org/wiki/Mira_Murati\n"
     ]
    }
   ],
   "source": [
    "for doc in documents:\n",
    "    print(doc.metadata['source'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "5c5d5a34-d079-4444-88f9-511da29fe523",
   "metadata": {},
   "outputs": [],
   "source": [
    "documents.remove(documents[2])\n",
    "documents.remove(documents[3])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "97cd6970-d1c4-444b-81bf-f2594c2b2045",
   "metadata": {},
   "source": [
    "# Enable Neo4j database"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "64657236-720e-4953-b598-8931eec0244d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install pypdf"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1c9550f5-cc18-46c2-9469-1f43991f3f1c",
   "metadata": {},
   "source": [
    "# News Articles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "aba1b836-437b-40ca-be38-3f1d11a42cc7",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.document_loaders import PyPDFLoader\n",
    "from langchain.docstore.document import Document\n",
    "import os\n",
    "\n",
    "# Directory containing your PDF files\n",
    "directory_path = '/Users/leannchen/Documents/tcnews'\n",
    "\n",
    "# Initialize PyPDFLoader for each PDF in the directory\n",
    "loaders = [PyPDFLoader(os.path.join(directory_path, f)) for f in os.listdir(directory_path) if f.endswith('.pdf')]\n",
    "\n",
    "# Load documents from PDFs\n",
    "news_docs = []\n",
    "for loader in loaders:\n",
    "    news_docs.extend(loader.load())\n",
    "\n",
    "# Prepare the content and metadata for each news article as Document objects\n",
    "news_articles_data = [\n",
    "    Document(\n",
    "        page_content=doc.page_content,  # Assuming this is how you access the page content of the document\n",
    "        metadata={\n",
    "            \"source\": doc.metadata['source'].removeprefix('/Users/leannchen/Documents/tcnews/'),  # Assuming this is the metadata format\n",
    "            # Include any other metadata items here\n",
    "        }\n",
    "    )\n",
    "    for doc in news_docs  # Assuming news_docs is a list of objects with page_content and metadata\n",
    "]\n",
    "\n",
    "# Later, when you are ready to add them to the database:\n",
    "# Call add_documents and construct Document objects inline\n",
    "# Assuming news_articles_data is already a list of Document objects\n",
    "# neo4j_db.add_documents(\n",
    "#     news_articles_data,\n",
    "#     ids=[f\"news_article_{i}\" for i in range(len(news_articles_data))]\n",
    "\n",
    "# )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2ce9e011-7bdf-438e-a591-9ad480ef78e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install spacy-llm\n",
    "# !pip install --upgrade jupyter ipywidgets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "710b9c7a-0cfa-4435-8cc7-aade9cfa2bc3",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_data = documents + news_articles_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ea91be3d-cc34-4093-921c-f55dd31ba652",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "12a944c3-8050-4c25-9ab1-37aa14ac8311",
   "metadata": {},
   "source": [
    "# Perform Article Summaries as Relationship Extraction Database"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 289,
   "id": "2528c928-2f38-42ed-a509-1ed4109885bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.chains.combine_documents.stuff import StuffDocumentsChain\n",
    "from langchain.chains.llm import LLMChain\n",
    "from langchain.chains import MapReduceDocumentsChain, ReduceDocumentsChain\n",
    "from langchain.prompts import PromptTemplate\n",
    "from langchain.llms import OpenAI\n",
    "from langchain.chat_models import ChatOpenAI\n",
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "# Initialize the text splitter\n",
    "rtext_splitter = RecursiveCharacterTextSplitter(chunk_size=1500, chunk_overlap=150)\n",
    "\n",
    "# Initialize LLM\n",
    "llm = ChatOpenAI(temperature=0, model_name=\"gpt-4\")\n",
    "\n",
    "# Define the map prompt template\n",
    "map_template = \"\"\"The following is a set of documents\n",
    "{all_data}\n",
    "Based on this list of docs, please perform concise summaries while extracting essential relationships for relationships analysis later, please do include dates of actions or events, which are very important for timeline analysis later. Example: \"Sam gets fired by the OpenAI board on 11/17/2023 or (Nov. 17th, Friday)\", which showcases not only the relationship between Sam and OpenAI, but also when it happens.\n",
    "Helpful Answer:\"\"\"\n",
    "map_prompt = PromptTemplate.from_template(map_template)\n",
    "\n",
    "# Define the map_chain\n",
    "map_chain = LLMChain(llm=llm, prompt=map_prompt)\n",
    "\n",
    "all_data = news_articles_data + documents\n",
    "# Extract text from each document\n",
    "# all_text_data = [doc.page_content for doc in all_data]\n",
    "\n",
    "# Reduce\n",
    "reduce_template = \"\"\"The following is set of summaries:\n",
    "{all_data}\n",
    "Take these and distill it into concise summaries of the articles while containing important relationships and events (including the timeline). Example: \"Sam gets fired by the OpenAI board on 11/17/2023 or (Nov. 17th, Friday)\", which showcases not only the relationship between Sam and OpenAI, but also when it happens.\n",
    "Helpful Answer:\"\"\"\n",
    "reduce_prompt = PromptTemplate.from_template(reduce_template)\n",
    "\n",
    "# ChatPromptTemplate(input_variables=['all_data'], messages=[HumanMessagePromptTemplate(prompt=PromptTemplate(input_variables=['all_data'], template='The following is a set of documents:\\n{all_data}\\nBased on this list of docs, please identify the main themes \\nHelpful Answer:'))])\n",
    "\n",
    "# Run chain\n",
    "reduce_chain = LLMChain(llm=llm, prompt=reduce_prompt)\n",
    "\n",
    "combine_documents_chain = StuffDocumentsChain(\n",
    "    llm_chain=reduce_chain,\n",
    "    document_variable_name=\"all_data\"  # This should match the variable name in reduce_prompt\n",
    ")\n",
    "\n",
    "# Combines and iteravely reduces the mapped documents\n",
    "reduce_documents_chain = ReduceDocumentsChain(\n",
    "    # This is final chain that is called.\n",
    "    combine_documents_chain=combine_documents_chain,\n",
    "    # If documents exceed context for `StuffDocumentsChain`\n",
    "    collapse_documents_chain=combine_documents_chain,\n",
    "    # The maximum number of tokens to group documents into.\n",
    "    token_max=4000,\n",
    ")\n",
    "\n",
    "# Combining documents by mapping a chain over them, then combining results\n",
    "map_reduce_chain = MapReduceDocumentsChain(\n",
    "    # Map chain\n",
    "    llm_chain=map_chain,\n",
    "    # Reduce chain\n",
    "    reduce_documents_chain=reduce_documents_chain,\n",
    "    # The variable name in the llm_chain to put the documents in\n",
    "    document_variable_name=\"all_data\",\n",
    "    # Return the results of the map steps in the output\n",
    "    return_intermediate_steps=False,\n",
    ")\n",
    "\n",
    "text_splitter = CharacterTextSplitter.from_tiktoken_encoder(\n",
    "    chunk_size=1000, chunk_overlap=0\n",
    ")\n",
    "split_docs = text_splitter.split_documents(all_data)\n",
    "\n",
    "# Run the MapReduce Chain\n",
    "summarization_results = map_reduce_chain.run(split_docs)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 290,
   "id": "5816cb1b-0a49-47fa-be60-102252980584",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"1. Sam Altman was fired as CEO of OpenAI on November 17, 2023, leading to a power struggle within the company. Over 730 employees threatened to quit and join Altman at Microsoft unless the board resigned and reappointed Altman and co-founder Greg Brockman. Despite initial talks of reinstatement, the board later confirmed that Altman would not be returning. Altman and Brockman subsequently joined Microsoft to head a new advanced AI research unit.\\n\\n2. Following Altman's departure, the OpenAI board underwent a reshuffle, with Mira Murati appointed as interim CEO, only to be later replaced by Emmett Shear, the former CEO of Twitch. The board's actions were criticized for lack of transparency and communication, with the exact reasons for Altman's removal remaining unclear.\\n\\n3. The OpenAI board, composed of Ilya Sutskever, Adam D’Angelo, Helen Toner, and Tasha McCauley, faced calls for resignation. Potential replacements included Bret Taylor and Will Hurd. The board's composition was criticized for lack of diversity and deep knowledge about responsible use of AI.\\n\\n4. Altman announced on November 20, 2023, that he would not be returning as CEO of OpenAI and would instead join Microsoft to lead a new AI research team. Nearly 500 of OpenAI’s roughly 770 employees threatened to resign unless the startup’s board resigned and reappointed Altman.\\n\\n5. Microsoft has been acquiring top executives and AI engineering talent from OpenAI, a generative AI company in which Microsoft holds a minority stake worth several billion dollars. Microsoft's leadership, particularly CEO Satya Nadella, has been projecting a 'business as usual' message during these upheavals at OpenAI.\\n\\n6. Microsoft successfully navigated through U.K. and EU competition authorities to merge with Activision by restructuring the deal and agreeing to certain conditions. However, its market power in cloud computing and potential influence over OpenAI is raising concerns among competition regulators.\\n\\n7. Emmett Shear, the ex-CEO of Twitch, was appointed as the interim CEO of OpenAI after Sam Altman was fired and replaced by CTO Mira Murati. Shear plans to hire an independent investigator to look into the events leading up to his appointment and reform the management and leadership team at OpenAI.\\n\\n8. OpenAI, an American AI research organization, was founded in December 2015. Microsoft invested $1 billion in OpenAI Global LLC in 2019 and $10 billion in 2023. On November 17, 2023, Sam Altman was removed as CEO and Greg Brockman was removed as chairman of OpenAI. Both returned four days later after negotiations with the board.\\n\\n9. Mira Murati, born in Albania in 1988, is a technology executive who has worked at Tesla, Leap Motion, and OpenAI. She served as the CTO of OpenAI from 2018, leading projects like ChatGPT, Dall-E, and Codex. She briefly served as interim CEO of OpenAI in November 2023 after Sam Altman was removed, but was replaced by Emmett Shear after three days.\\n\\n10. Sam Altman, born in 1985, is a technology entrepreneur who co-founded Loopt and Hydrazine Capital, and served as a partner and president at Y Combinator. He was the CEO of OpenAI from 2019 until his removal in November 2023, but was reinstated five days later. He also co-founded Tools For Humanity and raised $1 billion for OpenAI from Microsoft.\""
      ]
     },
     "execution_count": 290,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "summarization_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 291,
   "id": "a686f1a8-a3a7-4e4d-a297-3493927ff3f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Store summarization_results to a text file for future use\n",
    "# Timeline will further be added into the summaries\n",
    "with open('summary.txt', 'w') as file:\n",
    "    file.write(str(summarization_results))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dd4a39aa-5c78-49ed-a789-6514f33af1b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install spacy-llm"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9f8b2d01-5d61-41f5-a126-99194c2be22a",
   "metadata": {},
   "source": [
    "# Entity and Relationship"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 293,
   "id": "56098473-d023-4452-a439-4cf62556b9e8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Text: 1.\n",
      "Entities: [('1', 'CARDINAL')]\n",
      "Relations:\n",
      "Text: Sam Altman was fired as CEO of OpenAI on November 17, 2023, leading to a\n",
      "power struggle within the company.\n",
      "Entities: [('Sam Altman', 'PERSON'), ('November 17, 2023', 'DATE')]\n",
      "Relations:\n",
      "  - Sam Altman [fired_on] November 17, 2023\n",
      "Text: Over 730 employees threatened to quit and join Altman at Microsoft unless\n",
      "the board resigned and reappointed Altman and co-founder Greg Brockman.\n",
      "Entities: [('Over 730', 'CARDINAL'), ('Altman', 'ORG'), ('Microsoft', 'ORG'),\n",
      "('Altman', 'PERSON'), ('Greg Brockman', 'PERSON')]\n",
      "Relations:\n",
      "  - Greg Brockman [co-founder_of] Altman\n",
      "Text: Despite initial talks of reinstatement, the board later confirmed that\n",
      "Altman would not be returning.\n",
      "Entities: [('Altman', 'ORG')]\n",
      "Relations:\n",
      "Text: Altman and Brockman subsequently joined Microsoft to head a new advanced\n",
      "AI research unit.\n",
      "Entities: [('Altman', 'ORG'), ('Brockman', 'ORG'), ('Microsoft', 'ORG'), ('AI',\n",
      "'ORG')]\n",
      "Relations:\n",
      "  - Altman [joined] Microsoft\n",
      "  - Brockman [joined] Microsoft\n",
      "  - Altman [head] AI\n",
      "  - Brockman [head] AI\n",
      "Text: 2.\n",
      "Entities: [('2', 'CARDINAL')]\n",
      "Relations:\n",
      "Text: Following Altman's departure, the OpenAI board underwent a reshuffle, with\n",
      "Mira Murati appointed as interim CEO, only to be later replaced by Emmett Shear,\n",
      "the former CEO of Twitch.\n",
      "Entities: [('Altman', 'PERSON'), ('OpenAI', 'ORG'), ('Mira Murati', 'PERSON'),\n",
      "('Emmett Shear', 'PERSON'), ('Twitch', 'ORG')]\n",
      "Relations:\n",
      "  - Altman [departure_from] OpenAI\n",
      "  - Mira Murati [interim_CEO_of] OpenAI\n",
      "  - Emmett Shear [CEO_of] OpenAI\n",
      "  - Emmett Shear [former_CEO_of] Twitch\n",
      "Text: The board's actions were criticized for lack of transparency and\n",
      "communication, with the exact reasons for Altman's removal remaining unclear.\n",
      "Entities: [('Altman', 'ORG')]\n",
      "Relations:\n",
      "Text: 3.\n",
      "Entities: [('3', 'CARDINAL')]\n",
      "Relations:\n",
      "Text: The OpenAI board, composed of Ilya Sutskever, Adam D’Angelo, Helen Toner,\n",
      "and Tasha McCauley, faced calls for resignation.\n",
      "Entities: [('Ilya Sutskever', 'PERSON'), ('Adam D’Angelo', 'PERSON'), ('Helen\n",
      "Toner', 'PERSON'), ('Tasha McCauley', 'PERSON')]\n",
      "Relations:\n",
      "Text: Potential replacements included Bret Taylor and Will Hurd.\n",
      "Entities: [('Bret Taylor', 'PERSON'), ('Will Hurd', 'PERSON')]\n",
      "Relations:\n",
      "  - Bret Taylor [potential_replacements] Will Hurd\n",
      "Text: The board's composition was criticized for lack of diversity and deep\n",
      "knowledge about responsible use of AI.\n",
      "Entities: [('AI', 'ORG')]\n",
      "Relations:\n",
      "Text: 4.\n",
      "Entities: [('4', 'CARDINAL')]\n",
      "Relations:\n",
      "Text: Altman announced on November 20, 2023, that he would not be returning as\n",
      "CEO of OpenAI and would instead join Microsoft to lead a new AI research team.\n",
      "Entities: [('Altman', 'ORG'), ('November 20, 2023', 'DATE'), ('OpenAI', 'ORG'),\n",
      "('Microsoft', 'ORG'), ('AI', 'ORG')]\n",
      "Relations:\n",
      "  - Altman [lead] AI\n",
      "Text: Nearly 500 of OpenAI’s roughly 770 employees threatened to resign unless\n",
      "the startup’s board resigned and reappointed Altman.\n",
      "Entities: [('Nearly 500', 'CARDINAL'), ('roughly 770', 'CARDINAL'), ('Altman',\n",
      "'PERSON')]\n",
      "Relations:\n",
      "  - Nearly 500 [part_of] roughly 770\n",
      "  - Nearly 500 [threatened_by] Altman\n",
      "  - roughly 770 [threatened_by] Altman\n",
      "Text: 5.\n",
      "Entities: [('5', 'CARDINAL')]\n",
      "Relations:\n",
      "Text: Microsoft has been acquiring top executives and AI engineering talent from\n",
      "OpenAI, a generative AI company in which Microsoft holds a minority stake worth\n",
      "several billion dollars.\n",
      "Entities: [('Microsoft', 'ORG'), ('AI', 'ORG'), ('OpenAI', 'ORG'), ('Microsoft',\n",
      "'ORG'), ('several billion dollars', 'MONEY')]\n",
      "Relations:\n",
      "  - Microsoft [has_stake] several billion dollars\n",
      "Text: Microsoft's leadership, particularly CEO Satya Nadella, has been\n",
      "projecting a 'business as usual' message during these upheavals at OpenAI.  6.\n",
      "Entities: [('Microsoft', 'ORG'), ('Satya Nadella', 'PERSON'), ('6', 'CARDINAL')]\n",
      "Relations:\n",
      "  - Satya Nadella [works for] Microsoft\n",
      "Text: Microsoft successfully navigated through U.K. and EU competition\n",
      "authorities to merge with Activision by restructuring the deal and agreeing to\n",
      "certain conditions.\n",
      "Entities: [('Microsoft', 'ORG'), ('U.K.', 'GPE'), ('EU', 'ORG'), ('Activision',\n",
      "'ORG')]\n",
      "Relations:\n",
      "  - Microsoft [merge_with] Activision\n",
      "Text: However, its market power in cloud computing and potential influence over\n",
      "OpenAI is raising concerns among competition regulators.\n",
      "Entities: [('OpenAI', 'ORG')]\n",
      "Relations:\n",
      "Text: 7. Emmett Shear, the ex-CEO of Twitch, was appointed as the interim CEO of\n",
      "OpenAI after Sam Altman was fired and replaced by CTO Mira Murati.\n",
      "Entities: [('7', 'CARDINAL'), ('Emmett Shear', 'PERSON'), ('Twitch', 'ORG'),\n",
      "('Sam Altman', 'PERSON'), ('Mira Murati', 'PERSON')]\n",
      "Relations:\n",
      "  - Emmett Shear [ex-CEO_of] Twitch\n",
      "  - Emmett Shear [ordinal] 7\n",
      "Text: Shear plans to hire an independent investigator to look into the events\n",
      "leading up to his appointment and reform the management and leadership team at\n",
      "OpenAI.\n",
      "Entities: [('Shear', 'ORG')]\n",
      "Relations:\n",
      "Text: 8.\n",
      "Entities: [('8', 'CARDINAL')]\n",
      "Relations:\n",
      "Text: OpenAI, an American AI research organization, was founded in December\n",
      "2015.\n",
      "Entities: [('American', 'NORP'), ('December 2015', 'DATE')]\n",
      "Relations:\n",
      "  - American [founded_on] December 2015\n",
      "Text: Microsoft invested $1 billion in OpenAI Global LLC in 2019 and $10 billion\n",
      "in 2023.\n",
      "Entities: [('Microsoft', 'ORG'), ('$1 billion', 'MONEY'), ('OpenAI Global LLC',\n",
      "'ORG'), ('2019', 'DATE'), ('$10 billion', 'MONEY'), ('2023', 'DATE')]\n",
      "Relations:\n",
      "  - Microsoft [invested_in] $1 billion\n",
      "  - Microsoft [invested_in] OpenAI Global LLC\n",
      "  - Microsoft [invested_on] 2019\n",
      "  - Microsoft [invested_in] $10 billion\n",
      "  - Microsoft [invested_on] 2023\n",
      "Text: On November 17, 2023, Sam Altman was removed as CEO and Greg Brockman was\n",
      "removed as chairman of OpenAI.\n",
      "Entities: [('November 17, 2023', 'DATE'), ('Sam Altman', 'PERSON'), ('Greg\n",
      "Brockman', 'PERSON'), ('OpenAI', 'ORG')]\n",
      "Relations:\n",
      "  - Greg Brockman [role_in] OpenAI\n",
      "Text: Both returned four days later after negotiations with the board.\n",
      "Entities: [('four days later', 'DATE')]\n",
      "Relations:\n",
      "Text: 9.\n",
      "Entities: [('9', 'CARDINAL')]\n",
      "Relations:\n",
      "Text: Mira Murati, born in Albania in 1988, is a technology executive who has\n",
      "worked at Tesla, Leap Motion, and OpenAI.\n",
      "Entities: [('Mira Murati', 'PERSON'), ('Albania', 'GPE'), ('1988', 'DATE'),\n",
      "('Tesla', 'ORG'), ('Leap Motion', 'ORG')]\n",
      "Relations:\n",
      "  - Mira Murati [born_in] Albania\n",
      "  - Mira Murati [born_on] 1988\n",
      "  - Mira Murati [worked_at] Tesla\n",
      "  - Mira Murati [worked_at] Leap Motion\n",
      "Text: She served as the CTO of OpenAI from 2018, leading projects like ChatGPT,\n",
      "Dall-E, and Codex.\n",
      "Entities: [('OpenAI', 'ORG'), ('2018', 'DATE'), ('Dall', 'PERSON'), ('Codex',\n",
      "'ORG')]\n",
      "Relations:\n",
      "  - OpenAI [time] 2018\n",
      "  - Dall [affiliation] OpenAI\n",
      "  - Dall [time] 2018\n",
      "Text: She briefly served as interim CEO of OpenAI in November 2023 after Sam\n",
      "Altman was removed, but was replaced by Emmett Shear after three days.\n",
      "Entities: [('OpenAI', 'ORG'), ('November 2023', 'DATE'), ('Sam Altman',\n",
      "'PERSON'), ('Emmett Shear', 'PERSON'), ('three days', 'DATE')]\n",
      "Relations:\n",
      "  - OpenAI [time] November 2023\n",
      "  - Sam Altman [role] OpenAI\n",
      "  - Emmett Shear [role] OpenAI\n",
      "  - Emmett Shear [time] three days\n",
      "Text: 10.\n",
      "Entities: [('10', 'CARDINAL')]\n",
      "Relations:\n",
      "Text: Sam Altman, born in 1985, is a technology entrepreneur who co-founded\n",
      "Loopt and Hydrazine Capital, and served as a partner and president at Y\n",
      "Combinator.\n",
      "Entities: [('Sam Altman', 'PERSON'), ('1985', 'DATE'), ('Loopt and Hydrazine\n",
      "Capital', 'ORG'), ('Y Combinator', 'ORG')]\n",
      "Relations:\n",
      "  - Sam Altman [born_on] 1985\n",
      "  - Sam Altman [co-founded] Loopt and Hydrazine Capital\n",
      "  - Sam Altman [worked_at] Y Combinator\n",
      "Text: He was the CEO of OpenAI from 2019 until his removal in November 2023, but\n",
      "was reinstated five days later.\n",
      "Entities: [('OpenAI', 'ORG'), ('2019', 'DATE'), ('November 2023', 'DATE'),\n",
      "('five days later', 'DATE')]\n",
      "Relations:\n",
      "  - OpenAI [start_date] 2019\n",
      "  - OpenAI [end_date] November 2023\n",
      "  - OpenAI [reinstatement_date] five days later\n",
      "Text: He also co-founded Tools For Humanity and raised $1 billion for OpenAI\n",
      "from Microsoft.\n",
      "Entities: [('Tools For Humanity', 'ORG'), ('$1 billion', 'MONEY'), ('Microsoft',\n",
      "'ORG')]\n",
      "Relations:\n",
      "  - Tools For Humanity [raised_money_for] $1 billion\n",
      "  - $1 billion [invested_by] Microsoft\n",
      "Entity counts: Counter({'ORG': 39, 'PERSON': 24, 'DATE': 15, 'CARDINAL': 13,\n",
      "'MONEY': 4, 'GPE': 2, 'NORP': 1})\n",
      "Relation counts: Counter({'time': 4, 'invested_in': 3, 'worked_at': 3, 'joined':\n",
      "2, 'head': 2, 'threatened_by': 2, 'invested_on': 2, 'born_on': 2, 'role': 2,\n",
      "'fired_on': 1, 'co-founder_of': 1, 'departure_from': 1, 'interim_CEO_of': 1,\n",
      "'CEO_of': 1, 'former_CEO_of': 1, 'potential_replacements': 1, 'lead': 1,\n",
      "'part_of': 1, 'has_stake': 1, 'works for': 1, 'merge_with': 1, 'ex-CEO_of': 1,\n",
      "'ordinal': 1, 'founded_on': 1, 'role_in': 1, 'born_in': 1, 'affiliation': 1,\n",
      "'co-founded': 1, 'start_date': 1, 'end_date': 1, 'reinstatement_date': 1,\n",
      "'raised_money_for': 1, 'invested_by': 1})\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import json\n",
    "import spacy\n",
    "from collections import Counter\n",
    "from pathlib import Path\n",
    "from wasabi import msg\n",
    "from spacy_llm.util import assemble\n",
    "\n",
    "# traditional spacy NER (Named Recognition Library)\n",
    "def split_document_sent(text):\n",
    "    nlp = spacy.load(\"en_core_web_sm\")\n",
    "    doc = nlp(text)\n",
    "    return [sent.text.strip() for sent in doc.sents] # referencial\n",
    "\n",
    "# spacy-llm relationship extraction\n",
    "def process_text(nlp, text, verbose=False):\n",
    "    doc = nlp(text)\n",
    "    if verbose:\n",
    "        msg.text(f\"Text: {doc.text}\")\n",
    "        msg.text(f\"Entities: {[(ent.text, ent.label_) for ent in doc.ents]}\")\n",
    "        msg.text(\"Relations:\")\n",
    "        for r in doc._.rel:\n",
    "            msg.text(f\"  - {doc.ents[r.dep]} [{r.relation}] {doc.ents[r.dest]}\")\n",
    "    return doc\n",
    "\n",
    "def run_pipeline(config_path, examples_path=None, verbose=False):\n",
    "    if not os.getenv(\"OPENAI_API_KEY\"):\n",
    "        msg.fail(\"OPENAI_API_KEY env variable was not found. Set it and try again.\", exits=1)\n",
    "\n",
    "    nlp = assemble(config_path, overrides={} if examples_path is None else {\"paths.examples\": str(examples_path)})\n",
    "\n",
    "    # Initialize counters and storage\n",
    "    processed_data = []\n",
    "    entity_counts = Counter()\n",
    "    relation_counts = Counter()\n",
    "\n",
    "    # Load your articles and news data here\n",
    "    # all_data = news_articles_data + documents\n",
    "\n",
    "    sents = split_document_sent(summarization_results)\n",
    "    for sent in sents:\n",
    "        doc = process_text(nlp, sent, verbose)\n",
    "        entities = [(ent.text, ent.label_) for ent in doc.ents]\n",
    "        relations = [(doc.ents[r.dep].text, r.relation, doc.ents[r.dest].text) for r in doc._.rel]\n",
    "        \n",
    "        # Store processed data\n",
    "        processed_data.append({'text': doc.text, 'entities': entities, 'relations': relations})\n",
    "\n",
    "        # Update counters\n",
    "        entity_counts.update([ent[1] for ent in entities])\n",
    "        relation_counts.update([rel[1] for rel in relations])\n",
    "\n",
    "    # Export to JSON\n",
    "    with open('processed_data.json', 'w') as f:\n",
    "        json.dump(processed_data, f)\n",
    "\n",
    "    # Display summary\n",
    "    msg.text(f\"Entity counts: {entity_counts}\")\n",
    "    msg.text(f\"Relation counts: {relation_counts}\")\n",
    "\n",
    "# Set your configuration paths and flags\n",
    "config_path = Path(\"zeroshot.cfg\")\n",
    "examples_path = None  # or None if not using few-shot\n",
    "verbose = True\n",
    "\n",
    "# Run the pipeline\n",
    "file = run_pipeline(config_path, None, verbose)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}