From 5fa5df22ccc16bb943b71ed84ba777d6c37b3917 Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Fri, 18 Oct 2024 07:12:53 +0000 Subject: [PATCH] :sparkles: feat(docs): add file read tool, rename copy tool --- docs/src/components/BuiltinTools.mdx | 3 +- .../content/docs/reference/scripts/system.mdx | 32 +++++++++++-- packages/cli/src/docker.ts | 48 ++++++++++++------- packages/core/src/chat.ts | 1 + .../system.python_code_interpreter.genai.mjs | 29 ++++++++++- 5 files changed, 91 insertions(+), 22 deletions(-) diff --git a/docs/src/components/BuiltinTools.mdx b/docs/src/components/BuiltinTools.mdx index 8c704bc6dc..e0df464b06 100644 --- a/docs/src/components/BuiltinTools.mdx +++ b/docs/src/components/BuiltinTools.mdx @@ -37,7 +37,8 @@ import { LinkCard } from '@astrojs/starlight/components'; - + + diff --git a/docs/src/content/docs/reference/scripts/system.mdx b/docs/src/content/docs/reference/scripts/system.mdx index 0197bcd197..c0d4893a03 100644 --- a/docs/src/content/docs/reference/scripts/system.mdx +++ b/docs/src/content/docs/reference/scripts/system.mdx @@ -2601,7 +2601,8 @@ Python Dockerized code execution for data analysis - tool `python_code_interpreter_run`: Executes python 3.12 code for Data Analysis tasks in a docker container. The process output is returned. Do not generate visualizations. The only packages available are numpy, pandas, scipy. There is NO network connectivity. Do not attempt to install other packages or make web requests. You must copy all the necessary files or pass all the data because the python code runs in a separate container. -- tool `python_code_interpreter_copy_files`: Copy files from the host file system to the container file system. NO absolute paths. Returns the path of each file copied in the container. +- tool `python_code_interpreter_copy_files_to_container`: Copy files from the host file system to the container file system. NO absolute paths. Returns the path of each file copied in the container. +- tool `python_code_interpreter_read_file`: Reads a file from the container file system. No absolute paths. `````js wrap title="system.python_code_interpreter" system({ @@ -2646,7 +2647,7 @@ defTool( ) defTool( - "python_code_interpreter_copy_files", + "python_code_interpreter_copy_files_to_container", "Copy files from the host file system to the container file system. NO absolute paths. Returns the path of each file copied in the container.", { type: "object", @@ -2657,7 +2658,8 @@ defTool( }, toFolder: { type: "string", - description: "Container directory path. Not a filename.", + description: + "Container directory path. Default is '.' Not a filename.", }, }, required: ["from"], @@ -2669,6 +2671,30 @@ defTool( const res = await container.scheduler.add( async () => await container.copyTo(from, toFolder) ) + return res.join("\n") + } +) + +defTool( + "python_code_interpreter_read_file", + "Reads a file from the container file system. No absolute paths.", + { + type: "object", + properties: { + filename: { + type: "string", + description: "Container file path", + }, + }, + required: ["filename"], + }, + async (args) => { + const { context, filename } = args + context.log(`python: cat ${filename}`) + const container = await getContainer() + const res = await container.scheduler.add( + async () => await container.readText(filename) + ) return res } ) diff --git a/packages/cli/src/docker.ts b/packages/cli/src/docker.ts index c21a03458d..0baf169c41 100644 --- a/packages/cli/src/docker.ts +++ b/packages/cli/src/docker.ts @@ -24,6 +24,7 @@ import { isQuiet } from "./log" import Dockerode, { Container } from "dockerode" import { shellParse, shellQuote } from "../../core/src/shell" import { PLimitPromiseQueue } from "../../core/src/concurrency" +import { delay } from "es-toolkit" type DockerodeType = import("dockerode") @@ -145,7 +146,9 @@ export class DockerManager { filters, }) const info = containers?.[0] - if (info) return this._docker.getContainer(info.Id) + if (info) { + return this._docker.getContainer(info.Id) + } } catch {} return undefined } @@ -173,13 +176,14 @@ export class DockerManager { hostPath ) this.containers.push(c) + logVerbose(`container: resuming ${name}`) await c.resume() const st = await container.inspect() - if (st.State?.Status !== "running") { - logVerbose(`container: start failed`) - trace?.error(`container: start failed`) + const status = st.State?.Status + if (status !== "running") { + logVerbose(`container: start failed (${status})`) + trace?.error(`container: ${status}`) } - logVerbose(`container: resuming ${name}`) return c } } @@ -203,7 +207,7 @@ export class DockerManager { } = options let name = (userName || image).replace(/[^a-zA-Z0-9]+/g, "_") if (persistent) - name += `_${(await sha1string(JSON.stringify({ image, name, ports, env, networkEnabled, postCreateCommands }))).slice(0, 12)}` + name += `_${(await sha1string(JSON.stringify({ image, name, ports, env, networkEnabled, postCreateCommands, CORE_VERSION }))).slice(0, 12)}` else name += `_${randomHex(6)}` const hostPath = host.path.resolve( dotGenaiscriptPath(DOCKER_VOLUMES_DIR, name) @@ -324,16 +328,24 @@ export class DockerManager { const res = /^\//.test(to) ? host.path.resolve( hostPath, - DOCKER_CONTAINER_VOLUME, to.replace(/^\//, "") ) - : host.path.resolve(hostPath, DOCKER_CONTAINER_VOLUME, to || "") + : host.path.resolve(hostPath, to || "") return res } const resume: () => Promise = async () => { - const state = await container.inspect() - if (state.State.Paused) await container.unpause() + let state = await container.inspect() + if (state.State.Status === "paused") await container.unpause() + else if (state.State.Status === "exited") { + await container.start() + } else if (state.State.Status === "restarting") { + let retry = 0 + while (state.State.Restarting && retry++ < 5) { + await delay(1000) + state = await container.inspect() + } + } } const pause: () => Promise = async () => { @@ -363,17 +375,21 @@ export class DockerManager { } const { cwd: userCwd, label } = options || {} - const cwd = userCwd - ? resolveContainerPath(userCwd) - : "/" + DOCKER_CONTAINER_VOLUME + const cwd = "/" + host.path.join(DOCKER_CONTAINER_VOLUME, userCwd || ".") + try { - trace?.startDetails(`📦 ▶️ container exec ${label || command}`) + trace?.startDetails( + `📦 ▶️ container exec: ${userCwd || ""}> ${label || command}` + ) trace?.itemValue(`container`, container.id) trace?.itemValue(`cwd`, cwd) - trace?.fence(`${command} ${shellQuote(args || [])}`, "sh") + trace?.fence( + `${cwd}> ${command} ${shellQuote(args || [])}`, + "sh" + ) if (!isQuiet) logVerbose( - `container exec: ${shellQuote([command, ...args])}` + `container exec: ${userCwd || ""}> ${shellQuote([command, ...args])}` ) let inspection = await container.inspect() diff --git a/packages/core/src/chat.ts b/packages/core/src/chat.ts index ac95e32274..5ba84d607c 100644 --- a/packages/core/src/chat.ts +++ b/packages/core/src/chat.ts @@ -225,6 +225,7 @@ async function runToolCalls( output = await tool.impl({ context, ...args }) } catch (e) { logWarn(`tool: ${tool.spec.name} error`) + logError(e) trace.error(`tool: ${tool.spec.name} error`, e) output = errorMessage(e) } diff --git a/packages/core/src/genaisrc/system.python_code_interpreter.genai.mjs b/packages/core/src/genaisrc/system.python_code_interpreter.genai.mjs index 63309ca91a..961663aa8a 100644 --- a/packages/core/src/genaisrc/system.python_code_interpreter.genai.mjs +++ b/packages/core/src/genaisrc/system.python_code_interpreter.genai.mjs @@ -40,7 +40,7 @@ defTool( ) defTool( - "python_code_interpreter_copy_files", + "python_code_interpreter_copy_files_to_container", "Copy files from the host file system to the container file system. NO absolute paths. Returns the path of each file copied in the container.", { type: "object", @@ -51,7 +51,8 @@ defTool( }, toFolder: { type: "string", - description: "Container directory path. Not a filename.", + description: + "Container directory path. Default is '.' Not a filename.", }, }, required: ["from"], @@ -63,6 +64,30 @@ defTool( const res = await container.scheduler.add( async () => await container.copyTo(from, toFolder) ) + return res.join("\n") + } +) + +defTool( + "python_code_interpreter_read_file", + "Reads a file from the container file system. No absolute paths.", + { + type: "object", + properties: { + filename: { + type: "string", + description: "Container file path", + }, + }, + required: ["filename"], + }, + async (args) => { + const { context, filename } = args + context.log(`python: cat ${filename}`) + const container = await getContainer() + const res = await container.scheduler.add( + async () => await container.readText(filename) + ) return res } )