Python interpreter (#617)

* add python execution system prompt

* add csv tests

* rename sample

* smaller sample

* no cache

* use default python image

* use python:3 image

* remove spinnger from run

* pandas sample

* formatting

* escape requirements format

* improving description

* simplify requirements format

* pand test

* better intellisense for tools/system scripts

* format stdout/err in logs

docs/src/components/BuiltinTools.mdx

@@ -10,6 +10,7 @@ import { LinkCard } from '@astrojs/starlight/components';
 <LinkCard title="fs_read_file" description="Reads a file as text from the file system." href="/genaiscript/reference/scripts/system#systemfs_read_file" />
 <LinkCard title="fs_read_summary" description="Reads a summary of a file from the file system." href="/genaiscript/reference/scripts/system#systemfs_read_summary" />
 <LinkCard title="math_eval" description="Evaluates a math expression" href="/genaiscript/reference/scripts/system#systemmath" />
+<LinkCard title="python_interpreter" description="Executes python 3.12 code in a docker container. The process output is returned. Use 'print' to output data." href="/genaiscript/reference/scripts/system#systempython_interpreter" />
 <LinkCard title="retrieval_fuzz_search" description="Search for keywords using the full text of files and a fuzzy distance." href="/genaiscript/reference/scripts/system#systemretrieval_fuzz_search" />
 <LinkCard title="retrieval_vector_search" description="Search files using embeddings and similarity distance." href="/genaiscript/reference/scripts/system#systemretrieval_vector_search" />
 <LinkCard title="retrieval_web_search" description="Search the web for a user query using Bing Search." href="/genaiscript/reference/scripts/system#systemretrieval_web_search" />

docs/src/content/docs/reference/scripts/system.mdx

@@ -590,6 +590,73 @@ Emit type information compatible with PyLance.`
 `````
 
 
+### `system.python_interpreter`
+
+Python Dockerized code execution
+
+
+
+-  tool `python_interpreter`: Executes python 3.12 code in a docker container. The process output is returned. Use 'print' to output data.
+
+`````js wrap title="system.python_interpreter"
+system({
+    title: "Python Dockerized code execution",
+})
+
+const image = env.vars.pythonImage ?? "python:3.12"
+
+let container = null
+
+defTool(
+    "python_interpreter",
+    "Executes python 3.12 code in a docker container. The process output is returned. Use 'print' to output data.",
+    {
+        type: "object",
+        properties: {
+            requirements: {
+                type: "string",
+                description: `list of pip packages to install using pip. should be using the pip install format: 
+<package1>
+<package2>
+`,
+            },
+            main: {
+                type: "string",
+                description: "python 3.12 source code to execute",
+            },
+        },
+        required: ["requirements", "main"],
+    },
+    async (args) => {
+        const { requirements, main = "" } = args
+        console.log(`python: running code...`)
+        container = await host.container({ image, networkEnabled: true })
+        if (requirements) {
+            console.log(`installing: ` + requirements)
+            await container.writeText(
+                "requirements.txt",
+                requirements.replace(/[ ,]\s*/g, "\n")
+            )
+            const res = await container.exec("pip", [
+                "install",
+                "--root-user-action",
+                "ignore",
+                "-r",
+                "requirements.txt",
+            ])
+            if (res.failed) throw new Error(`Failed to install requirements`)
+        }
+
+        console.log(`code: ` + main)
+        await container.writeText("main.py", main)
+        const res = await container.exec("python", ["main.py"])
+        return res
+    }
+)
+
+`````
+
+
 ### `system.retrieval_fuzz_search`
 
 Full Text Fuzzy Search

packages/cli/src/docker.ts

@@ -13,8 +13,10 @@ import { errorMessage } from "../../core/src/error"
 import { host } from "../../core/src/host"
 import { installImport } from "../../core/src/import"
 import { TraceOptions } from "../../core/src/trace"
-import { logError, dotGenaiscriptPath } from "../../core/src/util"
+import { logError, dotGenaiscriptPath, logVerbose } from "../../core/src/util"
 import { CORE_VERSION } from "../../core/src/version"
+import { YAMLStringify } from "../../core/src/yaml"
+import { isQuiet } from "./log"
 
 type DockerodeType = import("dockerode")
 
@@ -62,6 +64,30 @@ export class DockerManager {
         this.containers = []
     }
 
+    async stopContainer(id: string) {
+        const c = await this._docker?.getContainer(id)
+        if (c) {
+            try {
+                await c.stop()
+            } catch {}
+            try {
+                await c.remove()
+            } catch (e) {
+                logError(e)
+            }
+        }
+        const i = this.containers.findIndex((c) => c.id === id)
+        if (i > -1) {
+            const container = this.containers[i]
+            try {
+                await remove(container.hostPath)
+            } catch (e) {
+                logError(e)
+            }
+            this.containers.splice(i, 1)
+        }
+    }
+
     async checkImage(image: string) {
         await this.init()
         try {
@@ -162,6 +188,10 @@ export class DockerManager {
             const inspection = await container.inspect()
             trace?.itemValue(`container state`, inspection.State?.Status)
 
+            const stop: () => Promise<void> = async () => {
+                await this.stopContainer(container.id)
+            }
+
             const exec: ShellHost["exec"] = async (
                 command,
                 args,
@@ -178,6 +208,10 @@ export class DockerManager {
                     trace?.itemValue(`container`, container.id)
                     trace?.itemValue(`cwd`, cwd)
                     trace?.item(`\`${command}\` ${args.join(" ")}`)
+                    if (!isQuiet)
+                        logVerbose(
+                            `container exec: ${command} ${args.join(" ")}`
+                        )
 
                     let inspection = await container.inspect()
                     trace?.itemValue(
@@ -220,8 +254,14 @@ export class DockerManager {
                         exitCode === 0,
                         `exit code: ${sres.exitCode}`
                     )
-                    if (sres.stdout) trace?.detailsFenced(`stdout`, sres.stdout)
-                    if (sres.stderr) trace?.detailsFenced(`stderr`, sres.stderr)
+                    if (sres.stdout) {
+                        trace?.detailsFenced(`stdout`, sres.stdout, "txt")
+                        if (!isQuiet) logVerbose(sres.stdout)
+                    }
+                    if (sres.stderr) {
+                        trace?.detailsFenced(`stderr`, sres.stderr, "txt")
+                        if (!isQuiet) logVerbose(sres.stderr)
+                    }
 
                     return sres
                 } catch (e) {
@@ -239,7 +279,9 @@ export class DockerManager {
             const writeText = async (filename: string, content: string) => {
                 const hostFilename = host.path.resolve(hostPath, filename)
                 await ensureDir(host.path.dirname(hostFilename))
-                await writeFile(hostFilename, content, { encoding: "utf8" })
+                await writeFile(hostFilename, content ?? "", {
+                    encoding: "utf8",
+                })
             }
 
             const readText = async (filename: string, content: string) => {
@@ -262,6 +304,7 @@ export class DockerManager {
                 disablePurge: !!options.disablePurge,
                 hostPath,
                 containerPath,
+                stop,
                 exec,
                 writeText,
                 readText,

packages/cli/src/run.ts

@@ -4,7 +4,6 @@ import { isQuiet } from "./log"
 import { emptyDir, ensureDir } from "fs-extra"
 import { convertDiagnosticsToSARIF } from "./sarif"
 import { buildProject } from "./build"
-import { createProgressSpinner } from "./spinner"
 import { diagnosticsToCSV } from "../../core/src/ast"
 import { CancellationOptions } from "../../core/src/cancellation"
 import { ChatCompletionsProgressReport } from "../../core/src/chattypes"
@@ -109,13 +108,8 @@ export async function runScript(
     const cancellationToken = options.cancellationToken
     const jsSource = options.jsSource
 
-    const spinner =
-        !stream && !isQuiet
-            ? createProgressSpinner(`preparing tools in ${process.cwd()}`)
-            : undefined
     const fail = (msg: string, exitCode: number) => {
-        if (spinner) spinner.fail(msg)
-        else logVerbose(msg)
+        logVerbose(msg)
         return { exitCode, result }
     }
 
@@ -130,7 +124,7 @@ export async function runScript(
             const ffs = await host.findFiles(arg, {
                 applyGitIgnore: excludeGitIgnore,
             })
-            if (!ffs.length) {
+            if (!ffs?.length) {
                 return fail(
                     `no files matching ${arg}`,
                     FILES_NOT_FOUND_ERROR_CODE
@@ -188,16 +182,15 @@ export async function runScript(
             infoCb: (args) => {
                 const { text } = args
                 if (text) {
-                    if (spinner) spinner.start(text)
-                    else if (!isQuiet) logVerbose(text)
+                    if (!isQuiet) logVerbose(text)
                     infoCb?.(args)
                 }
             },
             partialCb: (args) => {
                 const { responseChunk, tokensSoFar } = args
                 tokens = tokensSoFar
                 if (stream && responseChunk) process.stdout.write(responseChunk)
-                if (spinner) spinner.report({ count: tokens })
+                else if (!isQuiet) process.stderr.write(responseChunk)
                 partialCb?.(args)
             },
             skipLLM,
@@ -230,18 +223,13 @@ export async function runScript(
             },
         })
     } catch (err) {
-        if (spinner) spinner.fail()
         if (isCancelError(err))
             return fail("user cancelled", USER_CANCELLED_ERROR_CODE)
         logError(err)
         return fail("runtime error", RUNTIME_ERROR_CODE)
     }
     if (!isQuiet) logVerbose("") // force new line
-    if (spinner) {
-        if (result.status !== "success")
-            spinner.fail(`${spinner.text}, ${result.statusText}`)
-        else spinner.succeed()
-    } else if (result.status !== "success")
+    if (result.status !== "success")
         logVerbose(result.statusText ?? result.status)
 
     if (outTrace) await writeText(outTrace, trace.content)
@@ -434,7 +422,6 @@ export async function runScript(
     if (failOnErrors && result.annotations?.some((a) => a.severity === "error"))
         return fail("error annotations found", ANNOTATION_ERROR_CODE)
 
-    spinner?.stop()
     process.stderr.write("\n")
     return { exitCode: 0, result }
 }