diff --git a/.example.env b/.example.env index d2b71aa1..e90c9ab8 100644 --- a/.example.env +++ b/.example.env @@ -1,4 +1,4 @@ -FILE_HASH_TO_TOOLS='[["bafybeiepacjq6yjukta4gfiq2fyvsj4b4r2g7swtkb2qfjugbbtsjt47ya",["openai-text-davinci-002","openai-text-davinci-003","openai-gpt-3.5-turbo","openai-gpt-4"]],["bafybeiepc5v4ixwuu5m6p5stck5kf2ecgkydf6crj52i5umnl2qm5swb4i",["stabilityai-stable-diffusion-v1-5","stabilityai-stable-diffusion-xl-beta-v2-2-2","stabilityai-stable-diffusion-512-v2-1","stabilityai-stable-diffusion-768-v2-1"]],["bafybeidpbnqbruzqlq424qt3i5dcvyqmcimshjilftabnrroujmjhdmteu",["transfer-native"]],["bafybeicdi6noqhbmau6qa7xwvufet5fvp42wy5yokn5uggsbvojyyzhxxu",["prediction-offline","prediction-online"]]]' +FILE_HASH_TO_TOOLS='[["bafybeiepacjq6yjukta4gfiq2fyvsj4b4r2g7swtkb2qfjugbbtsjt47ya",["openai-text-davinci-002","openai-text-davinci-003","openai-gpt-3.5-turbo","openai-gpt-4"]],["bafybeiepc5v4ixwuu5m6p5stck5kf2ecgkydf6crj52i5umnl2qm5swb4i",["stabilityai-stable-diffusion-v1-5","stabilityai-stable-diffusion-xl-beta-v2-2-2","stabilityai-stable-diffusion-512-v2-1","stabilityai-stable-diffusion-768-v2-1"]],["bafybeidpbnqbruzqlq424qt3i5dcvyqmcimshjilftabnrroujmjhdmteu",["transfer-native"]],["bafybeif3izkobmvaoen23ine6tiqx55eaf4g3r56hdalnig656xivzpf3m",["prediction-offline","prediction-online"]]]' API_KEYS='[["openai","dummy_api_key"],["stabilityai","dummy_api_key"]]' ETHEREUM_LEDGER_RPC_0=https://rpc.eu-central-2.gateway.fm/v4/gnosis/non-archival/mainnet ETHEREUM_WEBSOCKET_RPC_0=wss://rpc.eu-central-2.gateway.fm/ws/v4/gnosis/non-archival/mainnet diff --git a/packages/packages.json b/packages/packages.json index 83c69122..d029b4b3 100644 --- a/packages/packages.json +++ b/packages/packages.json @@ -2,12 +2,12 @@ "dev": { "connection/valory/websocket_client/0.1.0": "bafybeicz53kzs5uvyiod2azntl76zwgmpgr22ven4wl5fnwt2m546j3wsu", "skill/valory/contract_subscription/0.1.0": "bafybeif3hkpgbzuoxsbqxnd752qkvk3onytltrufnyrphnqbi62si4mdhy", - "agent/valory/mech/0.1.0": "bafybeia3ja3njzgxseefhizeypzvi5epoe2467khymqywktcgwid5fljx4", + "agent/valory/mech/0.1.0": "bafybeifvlxhojvlnijphps2zed5pfp6it4uu374pigk43gabv4j33zifki", "skill/valory/multiplexer_abci/0.1.0": "bafybeiatryharf4xsxouas7dzofcjedxa3sx3ifm5wfzf5z5hgz3jps474", - "skill/valory/task_execution_abci/0.1.0": "bafybeiagkhgyrm3lbdmiufbb6fe2iuad6yvuavk4ksiy2tiasocv445ala", - "skill/valory/mech_abci/0.1.0": "bafybeifmlpw26kjfhpb36arpm242qtriuj63fd2pxvfpsrogytowxnarwa", + "skill/valory/task_execution_abci/0.1.0": "bafybeif5mfev2oczliptbsriddyhkpksp2ifrfq5sjap434wprq3o2ev2u", + "skill/valory/mech_abci/0.1.0": "bafybeicvppgseykjmb3ar2rciwgy34hqantledr55p2denvyhtrxwp7qeu", "contract/valory/agent_mech/0.1.0": "bafybeidrhnm7xfcbxaxyl26azjvd6ur7xkcr5pdvpbpk5tddpyma7c74xu", - "service/valory/mech/0.1.0": "bafybeifgfw7htwiitg4r6cwixeief5uezrziyrgudi2ynasgrcarzrzzua", + "service/valory/mech/0.1.0": "bafybeigvlszayc3tpfke6eicucaxqvvqrsyoghmzuqrqasw62t6usgkscm", "protocol/valory/acn_data_share/0.1.0": "bafybeieyixetwvz767zekhvg7r6etumyanzys6xbalx2brrfswybinnlhi", "protocol/valory/default/1.0.0": "bafybeiecmut3235aen7wxukllv424f3dysvvlgfmn562kzdunc5hdj3hxu" }, diff --git a/packages/valory/agents/mech/aea-config.yaml b/packages/valory/agents/mech/aea-config.yaml index de5a90ce..1f3f8b37 100644 --- a/packages/valory/agents/mech/aea-config.yaml +++ b/packages/valory/agents/mech/aea-config.yaml @@ -34,11 +34,11 @@ skills: - valory/abstract_abci:0.1.0:bafybeiff2dqnkeorf343fmb3547fhiznvdw4ewbwl4rqluek2cv6ibl5hi - valory/abstract_round_abci:0.1.0:bafybeiac62ennpw54gns2quk4g3yoaili2mb72nj6c52czobz5dcwj4mwi - valory/contract_subscription:0.1.0:bafybeif3hkpgbzuoxsbqxnd752qkvk3onytltrufnyrphnqbi62si4mdhy -- valory/mech_abci:0.1.0:bafybeifmlpw26kjfhpb36arpm242qtriuj63fd2pxvfpsrogytowxnarwa +- valory/mech_abci:0.1.0:bafybeicvppgseykjmb3ar2rciwgy34hqantledr55p2denvyhtrxwp7qeu - valory/multiplexer_abci:0.1.0:bafybeiatryharf4xsxouas7dzofcjedxa3sx3ifm5wfzf5z5hgz3jps474 - valory/registration_abci:0.1.0:bafybeib6fsfur5jnflcveidnaeylneybwazewufzwa5twnwovdqgwtwsxm - valory/reset_pause_abci:0.1.0:bafybeibqz7y3i4aepuprhijwdydkcsbqjtpeea6gdzpp5fgc6abrvjz25a -- valory/task_execution_abci:0.1.0:bafybeiagkhgyrm3lbdmiufbb6fe2iuad6yvuavk4ksiy2tiasocv445ala +- valory/task_execution_abci:0.1.0:bafybeif5mfev2oczliptbsriddyhkpksp2ifrfq5sjap434wprq3o2ev2u - valory/termination_abci:0.1.0:bafybeieb3gnvjxxsh73g67m7rivzknwb63xu4qeagpkv7f4mqz33ecikem - valory/transaction_settlement_abci:0.1.0:bafybeihdpac4ayfgop3ixflimlb3zzyeejlpqtljfptdak6tc7aq4u5fzi default_ledger: ethereum @@ -142,7 +142,7 @@ models: setup: all_participants: ${list:["0x10E867Ac2Fb0Aa156ca81eF440a5cdf373bE1AaC"]} safe_contract_address: ${str:0x5e1D1eb61E1164D5a50b28C575dA73A29595dFf7} - file_hash_to_tools_json: ${list:[["bafybeibi34bhbvesmvd6o24jxvuldrwen4wj62na3lhva7k4afkg2shinu",["openai-text-davinci-002","openai-text-davinci-003","openai-gpt-3.5-turbo","openai-gpt-4"]],["bafybeiafdm3jctiz6wwo3rmo3vdubk7j7l5tumoxi5n5rc3x452mtkgyua",["stabilityai-stable-diffusion-v1-5","stabilityai-stable-diffusion-xl-beta-v2-2-2","stabilityai-stable-diffusion-512-v2-1","stabilityai-stable-diffusion-768-v2-1"]],["bafybeidpbnqbruzqlq424qt3i5dcvyqmcimshjilftabnrroujmjhdmteu",["transfer-native"]]]} + file_hash_to_tools_json: ${list:[["bafybeibi34bhbvesmvd6o24jxvuldrwen4wj62na3lhva7k4afkg2shinu",["openai-text-davinci-002","openai-text-davinci-003","openai-gpt-3.5-turbo","openai-gpt-4"]],["bafybeiafdm3jctiz6wwo3rmo3vdubk7j7l5tumoxi5n5rc3x452mtkgyua",["stabilityai-stable-diffusion-v1-5","stabilityai-stable-diffusion-xl-beta-v2-2-2","stabilityai-stable-diffusion-512-v2-1","stabilityai-stable-diffusion-768-v2-1"]],["bafybeidpbnqbruzqlq424qt3i5dcvyqmcimshjilftabnrroujmjhdmteu",["transfer-native"]],["bafybeif3izkobmvaoen23ine6tiqx55eaf4g3r56hdalnig656xivzpf3m",["prediction-online","prediction-offline"]]]} api_keys_json: ${list:[["openai", "dummy_api_key"],["stabilityai", "dummy_api_key"]]} use_polling: ${bool:false} polling_interval: ${int:25} diff --git a/packages/valory/services/mech/service.yaml b/packages/valory/services/mech/service.yaml index cdd358f3..0788c646 100644 --- a/packages/valory/services/mech/service.yaml +++ b/packages/valory/services/mech/service.yaml @@ -7,7 +7,7 @@ license: Apache-2.0 fingerprint: README.md: bafybeif7ia4jdlazy6745ke2k2x5yoqlwsgwr6sbztbgqtwvs3ndm2p7ba fingerprint_ignore_patterns: [] -agent: valory/mech:0.1.0:bafybeia3ja3njzgxseefhizeypzvi5epoe2467khymqywktcgwid5fljx4 +agent: valory/mech:0.1.0:bafybeifvlxhojvlnijphps2zed5pfp6it4uu374pigk43gabv4j33zifki number_of_agents: 4 deployment: agent: diff --git a/packages/valory/skills/mech_abci/fsm_specification.yaml b/packages/valory/skills/mech_abci/fsm_specification.yaml index ad9ca991..a797b271 100644 --- a/packages/valory/skills/mech_abci/fsm_specification.yaml +++ b/packages/valory/skills/mech_abci/fsm_specification.yaml @@ -16,6 +16,7 @@ alphabet_in: - RESET_AND_PAUSE_TIMEOUT - RESET_TIMEOUT - ROUND_TIMEOUT +- TASK_EXECUTION_ROUND_TIMEOUT - SUSPICIOUS_ACTIVITY - VALIDATE_TIMEOUT - WAIT @@ -101,7 +102,7 @@ transition_func: (SynchronizeLateMessagesRound, SUSPICIOUS_ACTIVITY): TaskExecutionRound (TaskExecutionRound, DONE): RandomnessTransactionSubmissionRound (TaskExecutionRound, ERROR): MultiplexerRound - (TaskExecutionRound, ROUND_TIMEOUT): TaskExecutionRound + (TaskExecutionRound, TASK_EXECUTION_ROUND_TIMEOUT): TaskExecutionRound (ValidateTransactionRound, DONE): MultiplexerRound (ValidateTransactionRound, NEGATIVE): CheckTransactionHistoryRound (ValidateTransactionRound, NONE): SelectKeeperTransactionSubmissionBRound diff --git a/packages/valory/skills/mech_abci/skill.yaml b/packages/valory/skills/mech_abci/skill.yaml index 51f385a6..9f6c5fd5 100644 --- a/packages/valory/skills/mech_abci/skill.yaml +++ b/packages/valory/skills/mech_abci/skill.yaml @@ -10,7 +10,7 @@ fingerprint: behaviours.py: bafybeihgtg4l7qcu33ptyfn6cnohm3tcxlpkdqijyc5zjxmci6lqzxmogy composition.py: bafybeiektjao3czojlipbcj2oglrk4hsch76d7ft3zw3vpcerewvvi6idy dialogues.py: bafybeidhmgdnhxcgv35vahz3ycoiduug23kfyqvttqnywjp3eiuqal4bzy - fsm_specification.yaml: bafybeieeqdr76xgxqjdqalip7agguutwsn23n5xgop4ziach7l4hiqxvqm + fsm_specification.yaml: bafybeic4fbqrftemdseokvfiwwo7cyajloz2aapvzmqbubulc7r34wcpzm handlers.py: bafybeiffuduhg433qsu6lbet5jsaub63bzv2l4x756aj2fbnu5bnfu4ble models.py: bafybeic3miniq5lacarxspyxuvmlnwervwf2wmpnvw2gxpgk77fzyzcpvq fingerprint_ignore_patterns: [] @@ -22,7 +22,7 @@ skills: - valory/multiplexer_abci:0.1.0:bafybeiatryharf4xsxouas7dzofcjedxa3sx3ifm5wfzf5z5hgz3jps474 - valory/registration_abci:0.1.0:bafybeib6fsfur5jnflcveidnaeylneybwazewufzwa5twnwovdqgwtwsxm - valory/reset_pause_abci:0.1.0:bafybeibqz7y3i4aepuprhijwdydkcsbqjtpeea6gdzpp5fgc6abrvjz25a -- valory/task_execution_abci:0.1.0:bafybeiagkhgyrm3lbdmiufbb6fe2iuad6yvuavk4ksiy2tiasocv445ala +- valory/task_execution_abci:0.1.0:bafybeif5mfev2oczliptbsriddyhkpksp2ifrfq5sjap434wprq3o2ev2u - valory/termination_abci:0.1.0:bafybeieb3gnvjxxsh73g67m7rivzknwb63xu4qeagpkv7f4mqz33ecikem - valory/transaction_settlement_abci:0.1.0:bafybeihdpac4ayfgop3ixflimlb3zzyeejlpqtljfptdak6tc7aq4u5fzi behaviours: diff --git a/packages/valory/skills/task_execution_abci/fsm_specification.yaml b/packages/valory/skills/task_execution_abci/fsm_specification.yaml index 4bc69583..349015c8 100644 --- a/packages/valory/skills/task_execution_abci/fsm_specification.yaml +++ b/packages/valory/skills/task_execution_abci/fsm_specification.yaml @@ -1,7 +1,7 @@ alphabet_in: - DONE - ERROR -- ROUND_TIMEOUT +- TASK_EXECUTION_ROUND_TIMEOUT default_start_state: TaskExecutionRound final_states: - FinishedTaskExecutionRound @@ -16,4 +16,4 @@ states: transition_func: (TaskExecutionRound, DONE): FinishedTaskExecutionRound (TaskExecutionRound, ERROR): FinishedTaskExecutionWithErrorRound - (TaskExecutionRound, ROUND_TIMEOUT): TaskExecutionRound + (TaskExecutionRound, TASK_EXECUTION_ROUND_TIMEOUT): TaskExecutionRound diff --git a/packages/valory/skills/task_execution_abci/rounds.py b/packages/valory/skills/task_execution_abci/rounds.py index 832f65e7..dbf2d250 100644 --- a/packages/valory/skills/task_execution_abci/rounds.py +++ b/packages/valory/skills/task_execution_abci/rounds.py @@ -38,6 +38,7 @@ class Event(Enum): """TaskExecutionAbciApp Events""" + TASK_EXECUTION_ROUND_TIMEOUT = "task_execution_round_timeout" ROUND_TIMEOUT = "round_timeout" NO_MAJORITY = "no_majority" DONE = "done" @@ -117,7 +118,7 @@ class TaskExecutionAbciApp(AbciApp[Event]): Transition states: 0. TaskExecutionRound - done: 1. - - round timeout: 0. + - task execution round timeout: 0. - error: 2. 1. FinishedTaskExecutionRound 2. FinishedTaskExecutionWithErrorRound @@ -125,7 +126,7 @@ class TaskExecutionAbciApp(AbciApp[Event]): Final states: {FinishedTaskExecutionRound, FinishedTaskExecutionWithErrorRound} Timeouts: - round timeout: 30.0 + task execution round timeout: 60.0 """ initial_round_cls: AppState = TaskExecutionRound @@ -133,7 +134,7 @@ class TaskExecutionAbciApp(AbciApp[Event]): transition_function: AbciAppTransitionFunction = { TaskExecutionRound: { Event.DONE: FinishedTaskExecutionRound, - Event.ROUND_TIMEOUT: TaskExecutionRound, + Event.TASK_EXECUTION_ROUND_TIMEOUT: TaskExecutionRound, Event.ERROR: FinishedTaskExecutionWithErrorRound, }, FinishedTaskExecutionRound: {}, @@ -144,7 +145,7 @@ class TaskExecutionAbciApp(AbciApp[Event]): FinishedTaskExecutionWithErrorRound, } event_to_timeout: EventToTimeout = { - Event.ROUND_TIMEOUT: 30.0, + Event.TASK_EXECUTION_ROUND_TIMEOUT: 60.0, } cross_period_persisted_keys: FrozenSet[str] = frozenset() db_pre_conditions: Dict[AppState, Set[str]] = { diff --git a/packages/valory/skills/task_execution_abci/skill.yaml b/packages/valory/skills/task_execution_abci/skill.yaml index 96575168..1be8ddaf 100644 --- a/packages/valory/skills/task_execution_abci/skill.yaml +++ b/packages/valory/skills/task_execution_abci/skill.yaml @@ -10,13 +10,13 @@ fingerprint: __init__.py: bafybeihrkpey6kxur2uoimrskq2wfpelqidxeapdxie6iuv2x7dk77ksvu behaviours.py: bafybeiejpjkjvyvowefa32tuxzwtm2cazlgq33uiv6xzhjtkpo4rwumkde dialogues.py: bafybeibmac3m5u5h6ucoyjr4dazay72dyga656wvjl6z6saapluvjo54ne - fsm_specification.yaml: bafybeiggzywf6zpq2mb4ite6ptnb2zqcjjdhyu7jyb5npn7id4hcrm5tiy + fsm_specification.yaml: bafybeia66ok2ll4kjbbmgbocjfape6u6ctacgexrnpgmru6zudr5em7vty handlers.py: bafybeibe5n7my2vd2wlwo73sbma65epjqc7kxgtittewlylcmvnmoxtxzq io_/__init__.py: bafybeifxgmmwjqzezzn3e6keh2bfo4cyo7y5dq2ept3stfmgglbrzfl5rq io_/naive_loader.py: bafybeihqrt34jso7dwfcedh7itmmovfv55tdjhw2tkqifsbiohetbonynu models.py: bafybeihavofxq3nxt46x74idm2mjl5xxghoqzjtuxnx5i255k6mdwsyyaq payloads.py: bafybeigptsnusjowmqjcxnzc4ct7n2iczuiorlwqsg7dl6ipnwkjb6iqoe - rounds.py: bafybeig6gdswlwfpfvpnacpcy7vp7rt4y6cwxlmkl24wzyt4xhnd2z4zkq + rounds.py: bafybeifaza7nzpn7fv6xuk6pcamxne3b5tzqogricjkcvbek5cso2emcnm tasks.py: bafybeicu5t5cvfhbndgpxbbtmp4vbmtyb6fba6vsnlewftvuderxp5lwcy fingerprint_ignore_patterns: [] connections: diff --git a/tools/prediction_request.py b/tools/prediction_request.py index a8e979c9..ed612e89 100644 --- a/tools/prediction_request.py +++ b/tools/prediction_request.py @@ -20,26 +20,24 @@ """This module implements a Mech tool for binary predictions.""" import json -import random -from typing import Any, Dict, List, Optional, Tuple +import requests +from concurrent.futures import ThreadPoolExecutor +from typing import Any, Dict, List, Optional, Tuple, Generator import googlesearch import openai from bs4 import BeautifulSoup -from selenium import webdriver -from selenium.webdriver.common.by import By -from selenium.webdriver.firefox.options import Options NUM_URLS_EXTRACT = 5 - DEFAULT_OPENAI_SETTINGS = { "max_tokens": 500, "temperature": 0.7, } - -ALLOWED_TOOLS = ["prediction-offline", "prediction-online"] - +ALLOWED_TOOLS = [ + "prediction-offline", + "prediction-online", +] TOOL_TO_ENGINE = { "prediction-offline": "gpt-3.5-turbo", "prediction-online": "gpt-3.5-turbo", @@ -110,71 +108,120 @@ - "queries": An array of strings of size between 1 and 5. Each string must be a search engine query that can help obtain relevant information to estimate the probability that the event in "USER_PROMPT" occurs. You must provide original information in each query, and they should not overlap or lead to obtain the same set of results. - - "urls": An array of strings of size between 1 and 5. Each string must be a relevant URL that may contain information - about the event in "USER_PROMPT". The provided URLs must not repeat and must not be search engines. * Output only the JSON object. Do not include any other contents in your response. """ -def run(**kwargs) -> Tuple[str, Optional[Dict[str, Any]]]: - """Run the task""" - openai.api_key = kwargs["api_keys"]["openai"] - max_tokens = kwargs.get("max_tokens", DEFAULT_OPENAI_SETTINGS["max_tokens"]) - temperature = kwargs.get("temperature", DEFAULT_OPENAI_SETTINGS["temperature"]) - prompt = kwargs["prompt"] - tool = kwargs["tool"] - if tool not in ALLOWED_TOOLS: - raise ValueError(f"Tool {tool} is not supported.") +def get_urls_from_queries(queries: List[str]) -> List[str]: + """Get URLs from search engine queries""" + results = [] + for query in queries: + for url in googlesearch.search(query, num_results=3): + results.append(url) + unique_results = list(set(results)) + return unique_results - engine = TOOL_TO_ENGINE[tool] - additional_information = "" +def extract_text( + html: str, + num_words: int = 300, # TODO: summerise using GPT instead of limit +) -> str: + """Extract text from a single HTML document""" + soup = BeautifulSoup(html, "html.parser") + for script in soup(["script", "style"]): + script.extract() + text = soup.get_text() + lines = (line.strip() for line in text.splitlines()) + chunks = (phrase.strip() for line in lines for phrase in line.split(" ")) + text = "\n".join(chunk for chunk in chunks if chunk) + return text[:num_words] - if tool == "prediction-online": - url_query_prompt = URL_QUERY_PROMPT.format(user_prompt=prompt) - moderation_result = openai.Moderation.create(url_query_prompt) +def process_in_batches( + urls: List[str], window: int = 5 +) -> Generator[None, None, List[requests.Response]]: + """Iter URLs in batches.""" + with ThreadPoolExecutor() as executor: + for i in range(0, len(urls), window): + batch = urls[i : i + window] + yield executor.map(requests.get, batch) - if moderation_result["results"][0]["flagged"]: - return "Moderation flagged the prompt as in violation of terms." - messages = [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": url_query_prompt}, - ] - response = openai.ChatCompletion.create( - model=engine, - messages=messages, - temperature=temperature, - max_tokens=max_tokens, - n=1, - timeout=120, - stop=None, - ) +def extract_texts(urls: List[str], num_words: int = 300) -> List[str]: + """Extract texts from URLs""" + max_allowed = 5 + extracted_texts = [] + count = 0 + for batch in process_in_batches(urls=urls): + for result in batch: + if result.status_code != 200: + continue + extracted_texts.append(extract_text(html=result.text, num_words=num_words)) + count += 1 + if count >= max_allowed: + break + return extracted_texts + + +def fetch_additional_information( + prompt: str, + engine: str, + temperature: float, + max_tokens: int, +) -> str: + """Fetch additional information.""" + url_query_prompt = URL_QUERY_PROMPT.format(user_prompt=prompt) + moderation_result = openai.Moderation.create(url_query_prompt) + if moderation_result["results"][0]["flagged"]: + return "" + messages = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": url_query_prompt}, + ] + response = openai.ChatCompletion.create( + model=engine, + messages=messages, + temperature=temperature, + max_tokens=max_tokens, + n=1, + timeout=120, + stop=None, + ) + json_data = json.loads(response.choices[0].message.content) + urls = get_urls_from_queries(json_data["queries"]) + texts = extract_texts(urls) + return "\n".join(["- " + text for text in texts]) - json_data = json.loads(response.choices[0].message.content) - queries = json_data["queries"] - urls = json_data["urls"] - urls_from_queries = get_urls_from_queries(queries) - urls.extend(urls_from_queries) - url_texts = extract_texts(urls) - additional_information = "\n".join(["- " + text for text in url_texts]) +def run(**kwargs) -> Tuple[str, Optional[Dict[str, Any]]]: + """Run the task""" + tool = kwargs["tool"] + prompt = kwargs["prompt"] + max_tokens = kwargs.get("max_tokens", DEFAULT_OPENAI_SETTINGS["max_tokens"]) + temperature = kwargs.get("temperature", DEFAULT_OPENAI_SETTINGS["temperature"]) + openai.api_key = kwargs["api_keys"]["openai"] + if tool not in ALLOWED_TOOLS: + raise ValueError(f"Tool {tool} is not supported.") + + engine = TOOL_TO_ENGINE[tool] + additional_information = ( + fetch_additional_information( + prompt=prompt, engine=engine, temperature=temperature, max_tokens=max_tokens + ) + if tool == "prediction-online" + else "" + ) prediction_prompt = PREDICTION_PROMPT.format( user_prompt=prompt, additional_information=additional_information ) - moderation_result = openai.Moderation.create(prediction_prompt) - if moderation_result["results"][0]["flagged"]: - return "Moderation flagged the prompt as in violation of terms." - + return "Moderation flagged the prompt as in violation of terms.", None messages = [ {"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": prediction_prompt}, ] - response = openai.ChatCompletion.create( model=engine, messages=messages, @@ -184,53 +231,4 @@ def run(**kwargs) -> Tuple[str, Optional[Dict[str, Any]]]: timeout=120, stop=None, ) - return response.choices[0].message.content, None - - -def get_urls_from_queries(queries: List[str]) -> List[str]: - """Get URLs from search engine queries""" - results = [] - - for query in queries: - for url in googlesearch.search(query, num_results=3): - results.append(url) - - unique_results = list(set(results)) - return unique_results - - -def extract_texts(urls: List[str], num_words: int = 300) -> List[str]: - """Extract texts from URLs""" - selected_urls = random.sample(urls, NUM_URLS_EXTRACT) - extracted_texts = [] - - options = Options() - options.add_argument("-headless") - browser = webdriver.Firefox(options=options) - - def get_html(url: str) -> str: - """Get HTML content of a given URL""" - browser.get(url) - html = browser.find_element(By.TAG_NAME, "body").get_attribute("innerHTML") - return html - - def extract_text(html: str) -> str: - """Extract text from a single HTML document""" - soup = BeautifulSoup(html, "html.parser") - for script in soup(["script", "style"]): - script.extract() - text = soup.get_text() - lines = (line.strip() for line in text.splitlines()) - chunks = (phrase.strip() for line in lines for phrase in line.split(" ")) - text = "\n".join(chunk for chunk in chunks if chunk) - return text - - for url in selected_urls: - html = get_html(url) - text = extract_text(html) - words = text.split()[:num_words] - extracted_texts.append(" ".join(words)) - - browser.quit() - return extracted_texts