From 41260a98f4f0564e334771054fa371584061b937 Mon Sep 17 00:00:00 2001 From: pelikhan Date: Tue, 13 Aug 2024 06:03:17 -0700 Subject: [PATCH 01/16] add python execution system prompt --- docs/genaisrc/genaiscript.d.ts | 11 +++- docs/src/components/BuiltinTools.mdx | 1 + .../content/docs/reference/scripts/system.mdx | 57 +++++++++++++++++++ genaisrc/genaiscript.d.ts | 11 +++- packages/cli/src/docker.ts | 29 ++++++++++ packages/core/src/fetch.ts | 2 +- packages/core/src/genaisrc/genaiscript.d.ts | 11 +++- .../system.python_interpreter.genai.js | 45 +++++++++++++++ packages/core/src/types/prompt_template.d.ts | 5 ++ packages/sample/genaisrc/genaiscript.d.ts | 11 +++- .../sample/genaisrc/node/genaiscript.d.ts | 11 +++- .../genaisrc/python-container.genai.mjs | 8 +++ .../sample/genaisrc/python/genaiscript.d.ts | 11 +++- .../sample/genaisrc/style/genaiscript.d.ts | 11 +++- packages/sample/src/aici/genaiscript.d.ts | 11 +++- packages/sample/src/errors/genaiscript.d.ts | 11 +++- packages/sample/src/makecode/genaiscript.d.ts | 11 +++- packages/sample/src/tla/genaiscript.d.ts | 11 +++- packages/sample/src/vision/genaiscript.d.ts | 11 +++- slides/genaisrc/genaiscript.d.ts | 11 +++- 20 files changed, 263 insertions(+), 27 deletions(-) create mode 100644 packages/core/src/genaisrc/system.python_interpreter.genai.js create mode 100644 packages/sample/genaisrc/python-container.genai.mjs diff --git a/docs/genaisrc/genaiscript.d.ts b/docs/genaisrc/genaiscript.d.ts index 6e01d272c5..1ab1467b25 100644 --- a/docs/genaisrc/genaiscript.d.ts +++ b/docs/genaisrc/genaiscript.d.ts @@ -65,9 +65,9 @@ interface PromptLike extends PromptDefinition { text?: string } -type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" +type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" -type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" +type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" type FileMergeHandler = ( filename: string, @@ -210,6 +210,7 @@ interface ScriptRuntimeOptions { * - `system.functions`: use functions * - `system.math`: Math expression evaluator * - `system.python`: Expert at generating and understanding Python code. +* - `system.python_interpreter`: Python Dockerized code execution * - `system.retrieval_fuzz_search`: Full Text Fuzzy Search * - `system.retrieval_vector_search`: Embeddings Vector Search * - `system.retrieval_web_search`: Web Search @@ -230,6 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression +* - `python_interpreter`: Executes python code in a docker container * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. @@ -1622,6 +1624,11 @@ interface ContainerHost extends ShellHost { * @param toContainer directory in the container */ copyTo(fromHost: string | string[], toContainer: string): Promise + + /** + * Stops and cleans out the container + */ + stop(): Promise } interface PromptContext extends ChatGenerationContext { diff --git a/docs/src/components/BuiltinTools.mdx b/docs/src/components/BuiltinTools.mdx index acd6b4c9fc..99f6ce44d3 100644 --- a/docs/src/components/BuiltinTools.mdx +++ b/docs/src/components/BuiltinTools.mdx @@ -10,6 +10,7 @@ import { LinkCard } from '@astrojs/starlight/components'; + diff --git a/docs/src/content/docs/reference/scripts/system.mdx b/docs/src/content/docs/reference/scripts/system.mdx index 5772557e86..103af351a1 100644 --- a/docs/src/content/docs/reference/scripts/system.mdx +++ b/docs/src/content/docs/reference/scripts/system.mdx @@ -590,6 +590,63 @@ Emit type information compatible with PyLance.` ````` +### `system.python_interpreter` + +Python Dockerized code execution + + + +- tool `python_interpreter`: Executes python code in a docker container + +`````js wrap title="system.python_interpreter" +system({ + title: "Python Dockerized code execution", +}) + +const image = env.vars.pythonImage ?? "python:alpine" + +defTool( + "python_interpreter", + "Executes python code in a docker container", + { + type: "object", + properties: { + requirements: { + type: "string", + description: "pip install requirements.txt file to install", + }, + main: { + type: "string", + description: "python source code to execute", + }, + }, + required: ["requirements", "main"], + }, + async ({ requirements, main }) => { + const container = await host.container({ image }) + try { + if (requirements) { + await container.writeText("requirements.txt", requirements) + await container.exec("python", [ + "pip", + "install", + "-r", + "requirements.txt", + ]) + } + + await container.writeText("main.py", main) + const res = await container.exec("python", ["main.py"]) + return res + } finally { + await container.stop() + } + } +) + +````` + + ### `system.retrieval_fuzz_search` Full Text Fuzzy Search diff --git a/genaisrc/genaiscript.d.ts b/genaisrc/genaiscript.d.ts index 6e01d272c5..1ab1467b25 100644 --- a/genaisrc/genaiscript.d.ts +++ b/genaisrc/genaiscript.d.ts @@ -65,9 +65,9 @@ interface PromptLike extends PromptDefinition { text?: string } -type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" +type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" -type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" +type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" type FileMergeHandler = ( filename: string, @@ -210,6 +210,7 @@ interface ScriptRuntimeOptions { * - `system.functions`: use functions * - `system.math`: Math expression evaluator * - `system.python`: Expert at generating and understanding Python code. +* - `system.python_interpreter`: Python Dockerized code execution * - `system.retrieval_fuzz_search`: Full Text Fuzzy Search * - `system.retrieval_vector_search`: Embeddings Vector Search * - `system.retrieval_web_search`: Web Search @@ -230,6 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression +* - `python_interpreter`: Executes python code in a docker container * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. @@ -1622,6 +1624,11 @@ interface ContainerHost extends ShellHost { * @param toContainer directory in the container */ copyTo(fromHost: string | string[], toContainer: string): Promise + + /** + * Stops and cleans out the container + */ + stop(): Promise } interface PromptContext extends ChatGenerationContext { diff --git a/packages/cli/src/docker.ts b/packages/cli/src/docker.ts index ce19eaa1eb..81b20bea4c 100644 --- a/packages/cli/src/docker.ts +++ b/packages/cli/src/docker.ts @@ -62,6 +62,30 @@ export class DockerManager { this.containers = [] } + async stopContainer(id: string) { + const c = await this._docker?.getContainer(id) + if (c) { + try { + await c.stop() + } catch {} + try { + await c.remove() + } catch (e) { + logError(e) + } + } + const i = this.containers.findIndex((c) => c.id === id) + if (i > -1) { + const container = this.containers[i] + try { + await remove(container.hostPath) + } catch (e) { + logError(e) + } + this.containers.splice(i, 1) + } + } + async checkImage(image: string) { await this.init() try { @@ -162,6 +186,10 @@ export class DockerManager { const inspection = await container.inspect() trace?.itemValue(`container state`, inspection.State?.Status) + const stop: () => Promise = async () => { + await this.stopContainer(container.id) + } + const exec: ShellHost["exec"] = async ( command, args, @@ -262,6 +290,7 @@ export class DockerManager { disablePurge: !!options.disablePurge, hostPath, containerPath, + stop, exec, writeText, readText, diff --git a/packages/core/src/fetch.ts b/packages/core/src/fetch.ts index bcb20f1058..b46a5aa5fb 100644 --- a/packages/core/src/fetch.ts +++ b/packages/core/src/fetch.ts @@ -74,7 +74,7 @@ export function traceFetchPost( ${Object.entries(headers) .map(([k, v]) => `-H "${k}: ${v}" \\`) .join("\n")} --d '${JSON.stringify(body).replace(/'/g, "'\\''")}' +-d '${JSON.stringify(body).replace(/'/g, "'\\''").replace(/\r\n/g, "\n \\")}' ` if (trace) trace.detailsFenced(`✉️ fetch`, cmd, "bash") else logVerbose(cmd) diff --git a/packages/core/src/genaisrc/genaiscript.d.ts b/packages/core/src/genaisrc/genaiscript.d.ts index 6e01d272c5..1ab1467b25 100644 --- a/packages/core/src/genaisrc/genaiscript.d.ts +++ b/packages/core/src/genaisrc/genaiscript.d.ts @@ -65,9 +65,9 @@ interface PromptLike extends PromptDefinition { text?: string } -type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" +type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" -type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" +type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" type FileMergeHandler = ( filename: string, @@ -210,6 +210,7 @@ interface ScriptRuntimeOptions { * - `system.functions`: use functions * - `system.math`: Math expression evaluator * - `system.python`: Expert at generating and understanding Python code. +* - `system.python_interpreter`: Python Dockerized code execution * - `system.retrieval_fuzz_search`: Full Text Fuzzy Search * - `system.retrieval_vector_search`: Embeddings Vector Search * - `system.retrieval_web_search`: Web Search @@ -230,6 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression +* - `python_interpreter`: Executes python code in a docker container * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. @@ -1622,6 +1624,11 @@ interface ContainerHost extends ShellHost { * @param toContainer directory in the container */ copyTo(fromHost: string | string[], toContainer: string): Promise + + /** + * Stops and cleans out the container + */ + stop(): Promise } interface PromptContext extends ChatGenerationContext { diff --git a/packages/core/src/genaisrc/system.python_interpreter.genai.js b/packages/core/src/genaisrc/system.python_interpreter.genai.js new file mode 100644 index 0000000000..f1b7b6ff0e --- /dev/null +++ b/packages/core/src/genaisrc/system.python_interpreter.genai.js @@ -0,0 +1,45 @@ +system({ + title: "Python Dockerized code execution", +}) + +const image = env.vars.pythonImage ?? "python:alpine" + +defTool( + "python_interpreter", + "Executes python code in a docker container", + { + type: "object", + properties: { + requirements: { + type: "string", + description: "pip install requirements.txt file to install", + }, + main: { + type: "string", + description: "python source code to execute", + }, + }, + required: ["requirements", "main"], + }, + async ({ requirements, main }) => { + console.log(`python: running code...`) + const container = await host.container({ image }) + try { + if (requirements) { + await container.writeText("requirements.txt", requirements) + await container.exec("python", [ + "pip", + "install", + "-r", + "requirements.txt", + ]) + } + + await container.writeText("main.py", main) + const res = await container.exec("python", ["main.py"]) + return res + } finally { + await container.stop() + } + } +) diff --git a/packages/core/src/types/prompt_template.d.ts b/packages/core/src/types/prompt_template.d.ts index 8865999995..aad8e437d7 100644 --- a/packages/core/src/types/prompt_template.d.ts +++ b/packages/core/src/types/prompt_template.d.ts @@ -1587,6 +1587,11 @@ interface ContainerHost extends ShellHost { * @param toContainer directory in the container */ copyTo(fromHost: string | string[], toContainer: string): Promise + + /** + * Stops and cleans out the container + */ + stop(): Promise } interface PromptContext extends ChatGenerationContext { diff --git a/packages/sample/genaisrc/genaiscript.d.ts b/packages/sample/genaisrc/genaiscript.d.ts index 6e01d272c5..1ab1467b25 100644 --- a/packages/sample/genaisrc/genaiscript.d.ts +++ b/packages/sample/genaisrc/genaiscript.d.ts @@ -65,9 +65,9 @@ interface PromptLike extends PromptDefinition { text?: string } -type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" +type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" -type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" +type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" type FileMergeHandler = ( filename: string, @@ -210,6 +210,7 @@ interface ScriptRuntimeOptions { * - `system.functions`: use functions * - `system.math`: Math expression evaluator * - `system.python`: Expert at generating and understanding Python code. +* - `system.python_interpreter`: Python Dockerized code execution * - `system.retrieval_fuzz_search`: Full Text Fuzzy Search * - `system.retrieval_vector_search`: Embeddings Vector Search * - `system.retrieval_web_search`: Web Search @@ -230,6 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression +* - `python_interpreter`: Executes python code in a docker container * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. @@ -1622,6 +1624,11 @@ interface ContainerHost extends ShellHost { * @param toContainer directory in the container */ copyTo(fromHost: string | string[], toContainer: string): Promise + + /** + * Stops and cleans out the container + */ + stop(): Promise } interface PromptContext extends ChatGenerationContext { diff --git a/packages/sample/genaisrc/node/genaiscript.d.ts b/packages/sample/genaisrc/node/genaiscript.d.ts index 6e01d272c5..1ab1467b25 100644 --- a/packages/sample/genaisrc/node/genaiscript.d.ts +++ b/packages/sample/genaisrc/node/genaiscript.d.ts @@ -65,9 +65,9 @@ interface PromptLike extends PromptDefinition { text?: string } -type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" +type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" -type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" +type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" type FileMergeHandler = ( filename: string, @@ -210,6 +210,7 @@ interface ScriptRuntimeOptions { * - `system.functions`: use functions * - `system.math`: Math expression evaluator * - `system.python`: Expert at generating and understanding Python code. +* - `system.python_interpreter`: Python Dockerized code execution * - `system.retrieval_fuzz_search`: Full Text Fuzzy Search * - `system.retrieval_vector_search`: Embeddings Vector Search * - `system.retrieval_web_search`: Web Search @@ -230,6 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression +* - `python_interpreter`: Executes python code in a docker container * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. @@ -1622,6 +1624,11 @@ interface ContainerHost extends ShellHost { * @param toContainer directory in the container */ copyTo(fromHost: string | string[], toContainer: string): Promise + + /** + * Stops and cleans out the container + */ + stop(): Promise } interface PromptContext extends ChatGenerationContext { diff --git a/packages/sample/genaisrc/python-container.genai.mjs b/packages/sample/genaisrc/python-container.genai.mjs new file mode 100644 index 0000000000..d2afe0c843 --- /dev/null +++ b/packages/sample/genaisrc/python-container.genai.mjs @@ -0,0 +1,8 @@ +script({ + system: ["system", "system.python_interpreter"], + files: ["src/penguins.csv"], +}) + +const data = def("DATA", env.files, { sliceSample: 100 }) + +$`Analyze ${data} with a detailed statistical analysis.` diff --git a/packages/sample/genaisrc/python/genaiscript.d.ts b/packages/sample/genaisrc/python/genaiscript.d.ts index 6e01d272c5..1ab1467b25 100644 --- a/packages/sample/genaisrc/python/genaiscript.d.ts +++ b/packages/sample/genaisrc/python/genaiscript.d.ts @@ -65,9 +65,9 @@ interface PromptLike extends PromptDefinition { text?: string } -type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" +type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" -type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" +type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" type FileMergeHandler = ( filename: string, @@ -210,6 +210,7 @@ interface ScriptRuntimeOptions { * - `system.functions`: use functions * - `system.math`: Math expression evaluator * - `system.python`: Expert at generating and understanding Python code. +* - `system.python_interpreter`: Python Dockerized code execution * - `system.retrieval_fuzz_search`: Full Text Fuzzy Search * - `system.retrieval_vector_search`: Embeddings Vector Search * - `system.retrieval_web_search`: Web Search @@ -230,6 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression +* - `python_interpreter`: Executes python code in a docker container * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. @@ -1622,6 +1624,11 @@ interface ContainerHost extends ShellHost { * @param toContainer directory in the container */ copyTo(fromHost: string | string[], toContainer: string): Promise + + /** + * Stops and cleans out the container + */ + stop(): Promise } interface PromptContext extends ChatGenerationContext { diff --git a/packages/sample/genaisrc/style/genaiscript.d.ts b/packages/sample/genaisrc/style/genaiscript.d.ts index 6e01d272c5..1ab1467b25 100644 --- a/packages/sample/genaisrc/style/genaiscript.d.ts +++ b/packages/sample/genaisrc/style/genaiscript.d.ts @@ -65,9 +65,9 @@ interface PromptLike extends PromptDefinition { text?: string } -type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" +type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" -type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" +type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" type FileMergeHandler = ( filename: string, @@ -210,6 +210,7 @@ interface ScriptRuntimeOptions { * - `system.functions`: use functions * - `system.math`: Math expression evaluator * - `system.python`: Expert at generating and understanding Python code. +* - `system.python_interpreter`: Python Dockerized code execution * - `system.retrieval_fuzz_search`: Full Text Fuzzy Search * - `system.retrieval_vector_search`: Embeddings Vector Search * - `system.retrieval_web_search`: Web Search @@ -230,6 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression +* - `python_interpreter`: Executes python code in a docker container * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. @@ -1622,6 +1624,11 @@ interface ContainerHost extends ShellHost { * @param toContainer directory in the container */ copyTo(fromHost: string | string[], toContainer: string): Promise + + /** + * Stops and cleans out the container + */ + stop(): Promise } interface PromptContext extends ChatGenerationContext { diff --git a/packages/sample/src/aici/genaiscript.d.ts b/packages/sample/src/aici/genaiscript.d.ts index 6e01d272c5..1ab1467b25 100644 --- a/packages/sample/src/aici/genaiscript.d.ts +++ b/packages/sample/src/aici/genaiscript.d.ts @@ -65,9 +65,9 @@ interface PromptLike extends PromptDefinition { text?: string } -type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" +type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" -type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" +type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" type FileMergeHandler = ( filename: string, @@ -210,6 +210,7 @@ interface ScriptRuntimeOptions { * - `system.functions`: use functions * - `system.math`: Math expression evaluator * - `system.python`: Expert at generating and understanding Python code. +* - `system.python_interpreter`: Python Dockerized code execution * - `system.retrieval_fuzz_search`: Full Text Fuzzy Search * - `system.retrieval_vector_search`: Embeddings Vector Search * - `system.retrieval_web_search`: Web Search @@ -230,6 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression +* - `python_interpreter`: Executes python code in a docker container * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. @@ -1622,6 +1624,11 @@ interface ContainerHost extends ShellHost { * @param toContainer directory in the container */ copyTo(fromHost: string | string[], toContainer: string): Promise + + /** + * Stops and cleans out the container + */ + stop(): Promise } interface PromptContext extends ChatGenerationContext { diff --git a/packages/sample/src/errors/genaiscript.d.ts b/packages/sample/src/errors/genaiscript.d.ts index 6e01d272c5..1ab1467b25 100644 --- a/packages/sample/src/errors/genaiscript.d.ts +++ b/packages/sample/src/errors/genaiscript.d.ts @@ -65,9 +65,9 @@ interface PromptLike extends PromptDefinition { text?: string } -type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" +type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" -type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" +type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" type FileMergeHandler = ( filename: string, @@ -210,6 +210,7 @@ interface ScriptRuntimeOptions { * - `system.functions`: use functions * - `system.math`: Math expression evaluator * - `system.python`: Expert at generating and understanding Python code. +* - `system.python_interpreter`: Python Dockerized code execution * - `system.retrieval_fuzz_search`: Full Text Fuzzy Search * - `system.retrieval_vector_search`: Embeddings Vector Search * - `system.retrieval_web_search`: Web Search @@ -230,6 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression +* - `python_interpreter`: Executes python code in a docker container * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. @@ -1622,6 +1624,11 @@ interface ContainerHost extends ShellHost { * @param toContainer directory in the container */ copyTo(fromHost: string | string[], toContainer: string): Promise + + /** + * Stops and cleans out the container + */ + stop(): Promise } interface PromptContext extends ChatGenerationContext { diff --git a/packages/sample/src/makecode/genaiscript.d.ts b/packages/sample/src/makecode/genaiscript.d.ts index 6e01d272c5..1ab1467b25 100644 --- a/packages/sample/src/makecode/genaiscript.d.ts +++ b/packages/sample/src/makecode/genaiscript.d.ts @@ -65,9 +65,9 @@ interface PromptLike extends PromptDefinition { text?: string } -type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" +type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" -type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" +type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" type FileMergeHandler = ( filename: string, @@ -210,6 +210,7 @@ interface ScriptRuntimeOptions { * - `system.functions`: use functions * - `system.math`: Math expression evaluator * - `system.python`: Expert at generating and understanding Python code. +* - `system.python_interpreter`: Python Dockerized code execution * - `system.retrieval_fuzz_search`: Full Text Fuzzy Search * - `system.retrieval_vector_search`: Embeddings Vector Search * - `system.retrieval_web_search`: Web Search @@ -230,6 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression +* - `python_interpreter`: Executes python code in a docker container * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. @@ -1622,6 +1624,11 @@ interface ContainerHost extends ShellHost { * @param toContainer directory in the container */ copyTo(fromHost: string | string[], toContainer: string): Promise + + /** + * Stops and cleans out the container + */ + stop(): Promise } interface PromptContext extends ChatGenerationContext { diff --git a/packages/sample/src/tla/genaiscript.d.ts b/packages/sample/src/tla/genaiscript.d.ts index 6e01d272c5..1ab1467b25 100644 --- a/packages/sample/src/tla/genaiscript.d.ts +++ b/packages/sample/src/tla/genaiscript.d.ts @@ -65,9 +65,9 @@ interface PromptLike extends PromptDefinition { text?: string } -type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" +type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" -type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" +type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" type FileMergeHandler = ( filename: string, @@ -210,6 +210,7 @@ interface ScriptRuntimeOptions { * - `system.functions`: use functions * - `system.math`: Math expression evaluator * - `system.python`: Expert at generating and understanding Python code. +* - `system.python_interpreter`: Python Dockerized code execution * - `system.retrieval_fuzz_search`: Full Text Fuzzy Search * - `system.retrieval_vector_search`: Embeddings Vector Search * - `system.retrieval_web_search`: Web Search @@ -230,6 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression +* - `python_interpreter`: Executes python code in a docker container * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. @@ -1622,6 +1624,11 @@ interface ContainerHost extends ShellHost { * @param toContainer directory in the container */ copyTo(fromHost: string | string[], toContainer: string): Promise + + /** + * Stops and cleans out the container + */ + stop(): Promise } interface PromptContext extends ChatGenerationContext { diff --git a/packages/sample/src/vision/genaiscript.d.ts b/packages/sample/src/vision/genaiscript.d.ts index 6e01d272c5..1ab1467b25 100644 --- a/packages/sample/src/vision/genaiscript.d.ts +++ b/packages/sample/src/vision/genaiscript.d.ts @@ -65,9 +65,9 @@ interface PromptLike extends PromptDefinition { text?: string } -type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" +type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" -type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" +type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" type FileMergeHandler = ( filename: string, @@ -210,6 +210,7 @@ interface ScriptRuntimeOptions { * - `system.functions`: use functions * - `system.math`: Math expression evaluator * - `system.python`: Expert at generating and understanding Python code. +* - `system.python_interpreter`: Python Dockerized code execution * - `system.retrieval_fuzz_search`: Full Text Fuzzy Search * - `system.retrieval_vector_search`: Embeddings Vector Search * - `system.retrieval_web_search`: Web Search @@ -230,6 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression +* - `python_interpreter`: Executes python code in a docker container * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. @@ -1622,6 +1624,11 @@ interface ContainerHost extends ShellHost { * @param toContainer directory in the container */ copyTo(fromHost: string | string[], toContainer: string): Promise + + /** + * Stops and cleans out the container + */ + stop(): Promise } interface PromptContext extends ChatGenerationContext { diff --git a/slides/genaisrc/genaiscript.d.ts b/slides/genaisrc/genaiscript.d.ts index 6e01d272c5..1ab1467b25 100644 --- a/slides/genaisrc/genaiscript.d.ts +++ b/slides/genaisrc/genaiscript.d.ts @@ -65,9 +65,9 @@ interface PromptLike extends PromptDefinition { text?: string } -type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" +type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" -type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" +type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" type FileMergeHandler = ( filename: string, @@ -210,6 +210,7 @@ interface ScriptRuntimeOptions { * - `system.functions`: use functions * - `system.math`: Math expression evaluator * - `system.python`: Expert at generating and understanding Python code. +* - `system.python_interpreter`: Python Dockerized code execution * - `system.retrieval_fuzz_search`: Full Text Fuzzy Search * - `system.retrieval_vector_search`: Embeddings Vector Search * - `system.retrieval_web_search`: Web Search @@ -230,6 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression +* - `python_interpreter`: Executes python code in a docker container * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. @@ -1622,6 +1624,11 @@ interface ContainerHost extends ShellHost { * @param toContainer directory in the container */ copyTo(fromHost: string | string[], toContainer: string): Promise + + /** + * Stops and cleans out the container + */ + stop(): Promise } interface PromptContext extends ChatGenerationContext { From 4612878b1b324e284b3c4bdde0c7c802c52726a9 Mon Sep 17 00:00:00 2001 From: pelikhan Date: Tue, 13 Aug 2024 06:25:57 -0700 Subject: [PATCH 02/16] add csv tests --- .../content/docs/reference/scripts/system.mdx | 1 + packages/core/src/csv.test.ts | 86 ++++++++++++++++--- packages/core/src/csv.ts | 5 +- packages/core/src/liner.test.ts | 24 +++--- 4 files changed, 90 insertions(+), 26 deletions(-) diff --git a/docs/src/content/docs/reference/scripts/system.mdx b/docs/src/content/docs/reference/scripts/system.mdx index 103af351a1..660317e4c5 100644 --- a/docs/src/content/docs/reference/scripts/system.mdx +++ b/docs/src/content/docs/reference/scripts/system.mdx @@ -623,6 +623,7 @@ defTool( required: ["requirements", "main"], }, async ({ requirements, main }) => { + console.log(`python: running code...`) const container = await host.container({ image }) try { if (requirements) { diff --git a/packages/core/src/csv.test.ts b/packages/core/src/csv.test.ts index 1176b6669c..44df058a21 100644 --- a/packages/core/src/csv.test.ts +++ b/packages/core/src/csv.test.ts @@ -1,16 +1,74 @@ -import { CSVParse } from './csv' -import { describe, test } from 'node:test' -import assert from 'node:assert/strict' +import { describe, test, beforeEach } from "node:test"; +import assert from "node:assert/strict"; +import { CSVParse, CSVTryParse, CSVToMarkdown } from "./csv"; -describe('csv', () => { - test('parses simple CSV with default options', () => { - const csvText = 'name,age\nAlice,30\nBob,25' - const result = CSVParse(csvText) - assert.deepStrictEqual(result, [ - { name: 'Alice', age: '30' }, - { name: 'Bob', age: '25' } - ]) - }) +describe('CSVParse', () => { + test('Parse simple CSV data with default options', () => { + const csv = "name,age\nJohn,30\nJane,25"; + const result = CSVParse(csv); + assert.deepEqual(result, [ + { name: "John", age: "30" }, + { name: "Jane", age: "25" } + ]); + }); - // Additional tests for CSVParse... -}) + test('Parse CSV data with custom delimiter', () => { + const csv = "name|age\nJohn|30\nJane|25"; + const result = CSVParse(csv, { delimiter: "|" }); + assert.deepEqual(result, [ + { name: "John", age: "30" }, + { name: "Jane", age: "25" } + ]); + }); + + test('Parse CSV data with specified headers', () => { + const csv = "John,30\nJane,25"; + const result = CSVParse(csv, { headers: ["name", "age"] }); + assert.deepEqual(result, [ + { name: "John", age: "30" }, + { name: "Jane", age: "25" } + ]); + }); +}); + +describe('CSVTryParse', () => { + test('Try to parse valid CSV data', () => { + const csv = "name,age\nJohn,30\nJane,25"; + const result = CSVTryParse(csv); + assert.deepEqual(result, [ + { name: "John", age: "30" }, + { name: "Jane", age: "25" } + ]); + }); +}); + +describe('CSVToMarkdown', () => { + test('Convert parsed CSV data to markdown table', () => { + const csv = [{ name: "John", age: "30" }, { name: "Jane", age: "25" }]; + const result = CSVToMarkdown(csv); + const expected = ` +|name|age| +|-|-| +|John|30| +|Jane|25| +`.trim().replace(/[\t ]+/g, " "); + assert.equal(result, expected); + }); + + test('Convert parsed CSV data to markdown table with custom headers', () => { + const csv = [{ name: "John", age: "30" }, { name: "Jane", age: "25" }]; + const result = CSVToMarkdown(csv, { headers: ["age", "name"] }); + const expected = ` +|age|name| +|-|-| +|30|John| +|25|Jane| +`.trim().replace(/[\t ]+/g, " "); + assert.equal(result, expected); + }); + + test('Handle empty CSV data input', () => { + const result = CSVToMarkdown([]); + assert.equal(result, ""); + }); +}); \ No newline at end of file diff --git a/packages/core/src/csv.ts b/packages/core/src/csv.ts index 75d8093a46..87923ccec8 100644 --- a/packages/core/src/csv.ts +++ b/packages/core/src/csv.ts @@ -27,7 +27,7 @@ export function CSVTryParse( delimiter?: string headers?: string[] } & TraceOptions -): object[] { +): object[] | undefined { const { trace } = options || {} try { return CSVParse(text, options) @@ -51,8 +51,9 @@ export function CSVToMarkdown(csv: object[], options?: { headers?: string[] }) { ), ] const md = markdownTable(table, { - align: "left", stringLength: (str) => str.length, + padding: false, + alignDelimiters: false, }) // improves LLM performance const mdcompact = md.replace(/[\t ]+/g, " ") diff --git a/packages/core/src/liner.test.ts b/packages/core/src/liner.test.ts index 47154f43d1..5d267ecd51 100644 --- a/packages/core/src/liner.test.ts +++ b/packages/core/src/liner.test.ts @@ -47,8 +47,8 @@ index 8cf2f17f..e17283d9 100644 [4] +line 3 ` assertDiff(diff, expected) - }) - + }) + test("diff test 3", function () { const diff = `diff --git a/packages/core/src/liner.diff.txt b/packages/core/src/liner.diff.txt index 8cf2f17f..519f67a6 100644 @@ -71,7 +71,7 @@ index 8cf2f17f..519f67a6 100644 [4] line 3 ` assertDiff(diff, expected) - }) + }) test("diff test 4", function () { const diff = `diff --git a/packages/core/src/liner.ts b/packages/core/src/liner.ts @@ -100,7 +100,7 @@ index 1215f7e7..385884e0 100644 [37] } ` assertDiff(diff, expected) - }) + }) test("returns the original diff if it is empty", function () { const diff = "" const result = llmifyDiff(diff) @@ -109,10 +109,14 @@ index 1215f7e7..385884e0 100644 }) function assertDiff(diff: string, expected: string) { const result = llmifyDiff(diff) - console.log(diff) - console.log("\n> result") - console.log(result) - console.log("\n> expected") - console.log(expected) - assert.strictEqual(result, expected) + try { + assert.strictEqual(result, expected) + } catch (e) { + console.log(diff) + console.log("\n> result") + console.log(result) + console.log("\n> expected") + console.log(expected) + throw e + } } From af58d3641a0cfbfad751686b3850f7c50092f835 Mon Sep 17 00:00:00 2001 From: pelikhan Date: Tue, 13 Aug 2024 06:28:18 -0700 Subject: [PATCH 03/16] rename sample --- .../{python-container.genai.mjs => data-analyst.genai.mjs} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/sample/genaisrc/{python-container.genai.mjs => data-analyst.genai.mjs} (100%) diff --git a/packages/sample/genaisrc/python-container.genai.mjs b/packages/sample/genaisrc/data-analyst.genai.mjs similarity index 100% rename from packages/sample/genaisrc/python-container.genai.mjs rename to packages/sample/genaisrc/data-analyst.genai.mjs From 9a9abdde42a661d584d1fd72aba07b005a312f01 Mon Sep 17 00:00:00 2001 From: pelikhan Date: Tue, 13 Aug 2024 06:48:15 -0700 Subject: [PATCH 04/16] smaller sample --- .../core/src/genaisrc/system.python_interpreter.genai.js | 5 ++--- packages/sample/genaisrc/data-analyst.genai.mjs | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/core/src/genaisrc/system.python_interpreter.genai.js b/packages/core/src/genaisrc/system.python_interpreter.genai.js index f1b7b6ff0e..7def141f8b 100644 --- a/packages/core/src/genaisrc/system.python_interpreter.genai.js +++ b/packages/core/src/genaisrc/system.python_interpreter.genai.js @@ -6,7 +6,7 @@ const image = env.vars.pythonImage ?? "python:alpine" defTool( "python_interpreter", - "Executes python code in a docker container", + "Executes python code in a docker container. The process output is returned.", { type: "object", properties: { @@ -27,8 +27,7 @@ defTool( try { if (requirements) { await container.writeText("requirements.txt", requirements) - await container.exec("python", [ - "pip", + await container.exec("pip", [ "install", "-r", "requirements.txt", diff --git a/packages/sample/genaisrc/data-analyst.genai.mjs b/packages/sample/genaisrc/data-analyst.genai.mjs index d2afe0c843..a3e2906960 100644 --- a/packages/sample/genaisrc/data-analyst.genai.mjs +++ b/packages/sample/genaisrc/data-analyst.genai.mjs @@ -3,6 +3,6 @@ script({ files: ["src/penguins.csv"], }) -const data = def("DATA", env.files, { sliceSample: 100 }) +const data = def("DATA", env.files, { sliceSample: 25 }) $`Analyze ${data} with a detailed statistical analysis.` From bb96c01d089f997fcc5f16f7af4dc9127678fe5e Mon Sep 17 00:00:00 2001 From: pelikhan Date: Tue, 13 Aug 2024 06:49:42 -0700 Subject: [PATCH 05/16] no cache --- packages/core/src/genaisrc/system.python_interpreter.genai.js | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/core/src/genaisrc/system.python_interpreter.genai.js b/packages/core/src/genaisrc/system.python_interpreter.genai.js index 7def141f8b..d9d1fea2ce 100644 --- a/packages/core/src/genaisrc/system.python_interpreter.genai.js +++ b/packages/core/src/genaisrc/system.python_interpreter.genai.js @@ -29,6 +29,7 @@ defTool( await container.writeText("requirements.txt", requirements) await container.exec("pip", [ "install", + "--no-cache-dir", "-r", "requirements.txt", ]) From 3e012492676ab0c900f5b58d777e9db51837ee02 Mon Sep 17 00:00:00 2001 From: pelikhan Date: Tue, 13 Aug 2024 06:50:47 -0700 Subject: [PATCH 06/16] use default python image --- docs/genaisrc/genaiscript.d.ts | 2 +- docs/src/components/BuiltinTools.mdx | 2 +- docs/src/content/docs/reference/scripts/system.mdx | 10 +++++----- genaisrc/genaiscript.d.ts | 2 +- packages/core/src/genaisrc/genaiscript.d.ts | 2 +- .../src/genaisrc/system.python_interpreter.genai.js | 2 +- packages/sample/genaisrc/genaiscript.d.ts | 2 +- packages/sample/genaisrc/node/genaiscript.d.ts | 2 +- packages/sample/genaisrc/python/genaiscript.d.ts | 2 +- packages/sample/genaisrc/style/genaiscript.d.ts | 2 +- packages/sample/src/aici/genaiscript.d.ts | 2 +- packages/sample/src/errors/genaiscript.d.ts | 2 +- packages/sample/src/makecode/genaiscript.d.ts | 2 +- packages/sample/src/tla/genaiscript.d.ts | 2 +- packages/sample/src/vision/genaiscript.d.ts | 2 +- slides/genaisrc/genaiscript.d.ts | 2 +- 16 files changed, 20 insertions(+), 20 deletions(-) diff --git a/docs/genaisrc/genaiscript.d.ts b/docs/genaisrc/genaiscript.d.ts index 1ab1467b25..e618a36957 100644 --- a/docs/genaisrc/genaiscript.d.ts +++ b/docs/genaisrc/genaiscript.d.ts @@ -231,7 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression -* - `python_interpreter`: Executes python code in a docker container +* - `python_interpreter`: Executes python code in a docker container. The process output is returned. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. diff --git a/docs/src/components/BuiltinTools.mdx b/docs/src/components/BuiltinTools.mdx index 99f6ce44d3..0b3204b06e 100644 --- a/docs/src/components/BuiltinTools.mdx +++ b/docs/src/components/BuiltinTools.mdx @@ -10,7 +10,7 @@ import { LinkCard } from '@astrojs/starlight/components'; - + diff --git a/docs/src/content/docs/reference/scripts/system.mdx b/docs/src/content/docs/reference/scripts/system.mdx index 660317e4c5..8edc96ccb4 100644 --- a/docs/src/content/docs/reference/scripts/system.mdx +++ b/docs/src/content/docs/reference/scripts/system.mdx @@ -596,18 +596,18 @@ Python Dockerized code execution -- tool `python_interpreter`: Executes python code in a docker container +- tool `python_interpreter`: Executes python code in a docker container. The process output is returned. `````js wrap title="system.python_interpreter" system({ title: "Python Dockerized code execution", }) -const image = env.vars.pythonImage ?? "python:alpine" +const image = env.vars.pythonImage ?? "python" defTool( "python_interpreter", - "Executes python code in a docker container", + "Executes python code in a docker container. The process output is returned.", { type: "object", properties: { @@ -628,9 +628,9 @@ defTool( try { if (requirements) { await container.writeText("requirements.txt", requirements) - await container.exec("python", [ - "pip", + await container.exec("pip", [ "install", + "--no-cache-dir", "-r", "requirements.txt", ]) diff --git a/genaisrc/genaiscript.d.ts b/genaisrc/genaiscript.d.ts index 1ab1467b25..e618a36957 100644 --- a/genaisrc/genaiscript.d.ts +++ b/genaisrc/genaiscript.d.ts @@ -231,7 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression -* - `python_interpreter`: Executes python code in a docker container +* - `python_interpreter`: Executes python code in a docker container. The process output is returned. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. diff --git a/packages/core/src/genaisrc/genaiscript.d.ts b/packages/core/src/genaisrc/genaiscript.d.ts index 1ab1467b25..e618a36957 100644 --- a/packages/core/src/genaisrc/genaiscript.d.ts +++ b/packages/core/src/genaisrc/genaiscript.d.ts @@ -231,7 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression -* - `python_interpreter`: Executes python code in a docker container +* - `python_interpreter`: Executes python code in a docker container. The process output is returned. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. diff --git a/packages/core/src/genaisrc/system.python_interpreter.genai.js b/packages/core/src/genaisrc/system.python_interpreter.genai.js index d9d1fea2ce..659a6dbcef 100644 --- a/packages/core/src/genaisrc/system.python_interpreter.genai.js +++ b/packages/core/src/genaisrc/system.python_interpreter.genai.js @@ -2,7 +2,7 @@ system({ title: "Python Dockerized code execution", }) -const image = env.vars.pythonImage ?? "python:alpine" +const image = env.vars.pythonImage ?? "python" defTool( "python_interpreter", diff --git a/packages/sample/genaisrc/genaiscript.d.ts b/packages/sample/genaisrc/genaiscript.d.ts index 1ab1467b25..e618a36957 100644 --- a/packages/sample/genaisrc/genaiscript.d.ts +++ b/packages/sample/genaisrc/genaiscript.d.ts @@ -231,7 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression -* - `python_interpreter`: Executes python code in a docker container +* - `python_interpreter`: Executes python code in a docker container. The process output is returned. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. diff --git a/packages/sample/genaisrc/node/genaiscript.d.ts b/packages/sample/genaisrc/node/genaiscript.d.ts index 1ab1467b25..e618a36957 100644 --- a/packages/sample/genaisrc/node/genaiscript.d.ts +++ b/packages/sample/genaisrc/node/genaiscript.d.ts @@ -231,7 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression -* - `python_interpreter`: Executes python code in a docker container +* - `python_interpreter`: Executes python code in a docker container. The process output is returned. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. diff --git a/packages/sample/genaisrc/python/genaiscript.d.ts b/packages/sample/genaisrc/python/genaiscript.d.ts index 1ab1467b25..e618a36957 100644 --- a/packages/sample/genaisrc/python/genaiscript.d.ts +++ b/packages/sample/genaisrc/python/genaiscript.d.ts @@ -231,7 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression -* - `python_interpreter`: Executes python code in a docker container +* - `python_interpreter`: Executes python code in a docker container. The process output is returned. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. diff --git a/packages/sample/genaisrc/style/genaiscript.d.ts b/packages/sample/genaisrc/style/genaiscript.d.ts index 1ab1467b25..e618a36957 100644 --- a/packages/sample/genaisrc/style/genaiscript.d.ts +++ b/packages/sample/genaisrc/style/genaiscript.d.ts @@ -231,7 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression -* - `python_interpreter`: Executes python code in a docker container +* - `python_interpreter`: Executes python code in a docker container. The process output is returned. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. diff --git a/packages/sample/src/aici/genaiscript.d.ts b/packages/sample/src/aici/genaiscript.d.ts index 1ab1467b25..e618a36957 100644 --- a/packages/sample/src/aici/genaiscript.d.ts +++ b/packages/sample/src/aici/genaiscript.d.ts @@ -231,7 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression -* - `python_interpreter`: Executes python code in a docker container +* - `python_interpreter`: Executes python code in a docker container. The process output is returned. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. diff --git a/packages/sample/src/errors/genaiscript.d.ts b/packages/sample/src/errors/genaiscript.d.ts index 1ab1467b25..e618a36957 100644 --- a/packages/sample/src/errors/genaiscript.d.ts +++ b/packages/sample/src/errors/genaiscript.d.ts @@ -231,7 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression -* - `python_interpreter`: Executes python code in a docker container +* - `python_interpreter`: Executes python code in a docker container. The process output is returned. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. diff --git a/packages/sample/src/makecode/genaiscript.d.ts b/packages/sample/src/makecode/genaiscript.d.ts index 1ab1467b25..e618a36957 100644 --- a/packages/sample/src/makecode/genaiscript.d.ts +++ b/packages/sample/src/makecode/genaiscript.d.ts @@ -231,7 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression -* - `python_interpreter`: Executes python code in a docker container +* - `python_interpreter`: Executes python code in a docker container. The process output is returned. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. diff --git a/packages/sample/src/tla/genaiscript.d.ts b/packages/sample/src/tla/genaiscript.d.ts index 1ab1467b25..e618a36957 100644 --- a/packages/sample/src/tla/genaiscript.d.ts +++ b/packages/sample/src/tla/genaiscript.d.ts @@ -231,7 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression -* - `python_interpreter`: Executes python code in a docker container +* - `python_interpreter`: Executes python code in a docker container. The process output is returned. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. diff --git a/packages/sample/src/vision/genaiscript.d.ts b/packages/sample/src/vision/genaiscript.d.ts index 1ab1467b25..e618a36957 100644 --- a/packages/sample/src/vision/genaiscript.d.ts +++ b/packages/sample/src/vision/genaiscript.d.ts @@ -231,7 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression -* - `python_interpreter`: Executes python code in a docker container +* - `python_interpreter`: Executes python code in a docker container. The process output is returned. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. diff --git a/slides/genaisrc/genaiscript.d.ts b/slides/genaisrc/genaiscript.d.ts index 1ab1467b25..e618a36957 100644 --- a/slides/genaisrc/genaiscript.d.ts +++ b/slides/genaisrc/genaiscript.d.ts @@ -231,7 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression -* - `python_interpreter`: Executes python code in a docker container +* - `python_interpreter`: Executes python code in a docker container. The process output is returned. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. From 69015f47e2a00c0bf6999e73dc57a71aaea4bfc8 Mon Sep 17 00:00:00 2001 From: pelikhan Date: Tue, 13 Aug 2024 06:52:56 -0700 Subject: [PATCH 07/16] use python:3 image --- packages/core/src/genaisrc/system.python_interpreter.genai.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/src/genaisrc/system.python_interpreter.genai.js b/packages/core/src/genaisrc/system.python_interpreter.genai.js index 659a6dbcef..e5b7497fa0 100644 --- a/packages/core/src/genaisrc/system.python_interpreter.genai.js +++ b/packages/core/src/genaisrc/system.python_interpreter.genai.js @@ -2,7 +2,7 @@ system({ title: "Python Dockerized code execution", }) -const image = env.vars.pythonImage ?? "python" +const image = env.vars.pythonImage ?? "python:3" defTool( "python_interpreter", From c3f487c06ecbe1688a48ca757e07294574a4e3ab Mon Sep 17 00:00:00 2001 From: pelikhan Date: Tue, 13 Aug 2024 08:05:00 -0700 Subject: [PATCH 08/16] remove spinnger from run --- .../content/docs/reference/scripts/system.mdx | 39 ++++++++++--------- packages/cli/src/docker.ts | 18 +++++++-- packages/cli/src/run.ts | 21 ++-------- packages/core/src/chat.ts | 17 ++++---- .../system.python_interpreter.genai.js | 37 +++++++++--------- packages/core/src/json5.ts | 10 +++++ .../sample/genaisrc/data-analyst.genai.mjs | 5 ++- 7 files changed, 80 insertions(+), 67 deletions(-) diff --git a/docs/src/content/docs/reference/scripts/system.mdx b/docs/src/content/docs/reference/scripts/system.mdx index 8edc96ccb4..f9c24634d3 100644 --- a/docs/src/content/docs/reference/scripts/system.mdx +++ b/docs/src/content/docs/reference/scripts/system.mdx @@ -603,7 +603,9 @@ system({ title: "Python Dockerized code execution", }) -const image = env.vars.pythonImage ?? "python" +const image = env.vars.pythonImage ?? "python:3" + +let container = null defTool( "python_interpreter", @@ -622,26 +624,25 @@ defTool( }, required: ["requirements", "main"], }, - async ({ requirements, main }) => { + async (args) => { + const { requirements, main = "" } = args console.log(`python: running code...`) - const container = await host.container({ image }) - try { - if (requirements) { - await container.writeText("requirements.txt", requirements) - await container.exec("pip", [ - "install", - "--no-cache-dir", - "-r", - "requirements.txt", - ]) - } - - await container.writeText("main.py", main) - const res = await container.exec("python", ["main.py"]) - return res - } finally { - await container.stop() + container = await host.container({ image }) + if (requirements) { + console.log(`installing: ` + requirements.replace(/\n/g, ", ")) + await container.writeText("requirements.txt", requirements) + await container.exec("pip", [ + "install", + "--no-cache-dir", + "-r", + "requirements.txt", + ]) } + + console.log(`code: ` + main) + await container.writeText("main.py", main) + const res = await container.exec("python", ["main.py"]) + return res } ) diff --git a/packages/cli/src/docker.ts b/packages/cli/src/docker.ts index 81b20bea4c..d1e15183c7 100644 --- a/packages/cli/src/docker.ts +++ b/packages/cli/src/docker.ts @@ -13,8 +13,10 @@ import { errorMessage } from "../../core/src/error" import { host } from "../../core/src/host" import { installImport } from "../../core/src/import" import { TraceOptions } from "../../core/src/trace" -import { logError, dotGenaiscriptPath } from "../../core/src/util" +import { logError, dotGenaiscriptPath, logVerbose } from "../../core/src/util" import { CORE_VERSION } from "../../core/src/version" +import { YAMLStringify } from "../../core/src/yaml" +import { isQuiet } from "./log" type DockerodeType = import("dockerode") @@ -248,8 +250,14 @@ export class DockerManager { exitCode === 0, `exit code: ${sres.exitCode}` ) - if (sres.stdout) trace?.detailsFenced(`stdout`, sres.stdout) - if (sres.stderr) trace?.detailsFenced(`stderr`, sres.stderr) + if (sres.stdout) { + trace?.detailsFenced(`stdout`, sres.stdout) + if (!isQuiet) logVerbose(sres.stdout) + } + if (sres.stderr) { + trace?.detailsFenced(`stderr`, sres.stderr) + if (!isQuiet) logVerbose(sres.stdout) + } return sres } catch (e) { @@ -267,7 +275,9 @@ export class DockerManager { const writeText = async (filename: string, content: string) => { const hostFilename = host.path.resolve(hostPath, filename) await ensureDir(host.path.dirname(hostFilename)) - await writeFile(hostFilename, content, { encoding: "utf8" }) + await writeFile(hostFilename, content ?? "", { + encoding: "utf8", + }) } const readText = async (filename: string, content: string) => { diff --git a/packages/cli/src/run.ts b/packages/cli/src/run.ts index 66ff6b4eb7..d894d65cbf 100644 --- a/packages/cli/src/run.ts +++ b/packages/cli/src/run.ts @@ -4,7 +4,6 @@ import { isQuiet } from "./log" import { emptyDir, ensureDir } from "fs-extra" import { convertDiagnosticsToSARIF } from "./sarif" import { buildProject } from "./build" -import { createProgressSpinner } from "./spinner" import { diagnosticsToCSV } from "../../core/src/ast" import { CancellationOptions } from "../../core/src/cancellation" import { ChatCompletionsProgressReport } from "../../core/src/chattypes" @@ -109,13 +108,8 @@ export async function runScript( const cancellationToken = options.cancellationToken const jsSource = options.jsSource - const spinner = - !stream && !isQuiet - ? createProgressSpinner(`preparing tools in ${process.cwd()}`) - : undefined const fail = (msg: string, exitCode: number) => { - if (spinner) spinner.fail(msg) - else logVerbose(msg) + logVerbose(msg) return { exitCode, result } } @@ -188,8 +182,7 @@ export async function runScript( infoCb: (args) => { const { text } = args if (text) { - if (spinner) spinner.start(text) - else if (!isQuiet) logVerbose(text) + if (!isQuiet) logVerbose(text) infoCb?.(args) } }, @@ -197,7 +190,7 @@ export async function runScript( const { responseChunk, tokensSoFar } = args tokens = tokensSoFar if (stream && responseChunk) process.stdout.write(responseChunk) - if (spinner) spinner.report({ count: tokens }) + else if (!isQuiet) process.stderr.write(responseChunk) partialCb?.(args) }, skipLLM, @@ -230,18 +223,13 @@ export async function runScript( }, }) } catch (err) { - if (spinner) spinner.fail() if (isCancelError(err)) return fail("user cancelled", USER_CANCELLED_ERROR_CODE) logError(err) return fail("runtime error", RUNTIME_ERROR_CODE) } if (!isQuiet) logVerbose("") // force new line - if (spinner) { - if (result.status !== "success") - spinner.fail(`${spinner.text}, ${result.statusText}`) - else spinner.succeed() - } else if (result.status !== "success") + if (result.status !== "success") logVerbose(result.statusText ?? result.status) if (outTrace) await writeText(outTrace, trace.content) @@ -434,7 +422,6 @@ export async function runScript( if (failOnErrors && result.annotations?.some((a) => a.severity === "error")) return fail("error annotations found", ANNOTATION_ERROR_CODE) - spinner?.stop() process.stderr.write("\n") return { exitCode: 0, result } } diff --git a/packages/core/src/chat.ts b/packages/core/src/chat.ts index 54fabbf167..fab53c91fb 100644 --- a/packages/core/src/chat.ts +++ b/packages/core/src/chat.ts @@ -2,13 +2,13 @@ import { MarkdownTrace } from "./trace" import { PromptImage, renderPromptNode } from "./promptdom" import { LanguageModelConfiguration, host } from "./host" import { GenerationOptions } from "./generation" -import { JSON5TryParse, JSON5parse, isJSONObjectOrArray } from "./json5" +import { JSON5parse, JSONLLMTryParse, isJSONObjectOrArray } from "./json5" import { CancellationOptions, CancellationToken, checkCancelled, } from "./cancellation" -import { assert } from "./util" +import { assert, logError } from "./util" import { extractFenced, findFirstDataFence } from "./fence" import { toStrictJSONSchema, @@ -18,7 +18,6 @@ import { import { MAX_DATA_REPAIRS, MAX_TOOL_CALLS } from "./constants" import { parseAnnotations } from "./annotations" import { errorMessage, isCancelError, serializeError } from "./error" -import { YAMLStringify } from "./yaml" import { estimateChatTokens } from "./chatencoder" import { createChatTurnGenerationContext } from "./runpromptcontext" import { dedent } from "./indent" @@ -141,10 +140,11 @@ async function runToolCalls( checkCancelled(cancellationToken) trace.startDetails(`📠 tool call ${call.name}`) try { - const callArgs: any = call.arguments - ? JSON5TryParse(call.arguments) + const callArgs: any = call.arguments // sometimes wrapped in \`\`\`json ... + ? JSONLLMTryParse(call.arguments) : undefined - trace.itemValue(`args`, callArgs ?? call.arguments) + trace.fence(call.arguments, "json") + if (callArgs === undefined) trace.error("arguments failed to parse") const fd = functions.find((f) => f.definition.name === call.name) if (!fd) throw new Error(`tool ${call.name} not found`) @@ -207,6 +207,7 @@ ${fenceMD(content, " ")} tool_call_id: call.id, }) } catch (e) { + logError(e) trace.error(`tool call ${call.id} error`, e) throw e } finally { @@ -238,7 +239,7 @@ async function applyRepairs( const invalids = fences.filter((f) => f?.validation?.valid === false) if (responseSchema) { - const value = JSON5TryParse(content) + const value = JSONLLMTryParse(content) const schema = promptParametersSchemaToJSONSchema(responseSchema) const res = validateJSONWithSchema(value, schema, { trace }) if (!res.valid) @@ -347,7 +348,7 @@ function structurifyChatSession( } } else { json = isJSONObjectOrArray(text) - ? JSON5TryParse(text, undefined) + ? JSONLLMTryParse(text) : (undefined ?? findFirstDataFence(fences)) } const frames: DataFrame[] = [] diff --git a/packages/core/src/genaisrc/system.python_interpreter.genai.js b/packages/core/src/genaisrc/system.python_interpreter.genai.js index e5b7497fa0..540ad5878f 100644 --- a/packages/core/src/genaisrc/system.python_interpreter.genai.js +++ b/packages/core/src/genaisrc/system.python_interpreter.genai.js @@ -4,6 +4,8 @@ system({ const image = env.vars.pythonImage ?? "python:3" +let container = null + defTool( "python_interpreter", "Executes python code in a docker container. The process output is returned.", @@ -21,25 +23,24 @@ defTool( }, required: ["requirements", "main"], }, - async ({ requirements, main }) => { + async (args) => { + const { requirements, main = "" } = args console.log(`python: running code...`) - const container = await host.container({ image }) - try { - if (requirements) { - await container.writeText("requirements.txt", requirements) - await container.exec("pip", [ - "install", - "--no-cache-dir", - "-r", - "requirements.txt", - ]) - } - - await container.writeText("main.py", main) - const res = await container.exec("python", ["main.py"]) - return res - } finally { - await container.stop() + container = await host.container({ image }) + if (requirements) { + console.log(`installing: ` + requirements.replace(/\n/g, ", ")) + await container.writeText("requirements.txt", requirements) + await container.exec("pip", [ + "install", + "--no-cache-dir", + "-r", + "requirements.txt", + ]) } + + console.log(`code: ` + main) + await container.writeText("main.py", main) + const res = await container.exec("python", ["main.py"]) + return res } ) diff --git a/packages/core/src/json5.ts b/packages/core/src/json5.ts index d28d877455..5d60d5aadc 100644 --- a/packages/core/src/json5.ts +++ b/packages/core/src/json5.ts @@ -52,3 +52,13 @@ export function JSON5TryParse( repair: true, }) } + +const startRx = /^\s*\`\`\`json\s/ +const endRx = /\`\`\`\s*$/ +export function JSONLLMTryParse(s: string): any { + if (s === undefined || s === null) return s + if (startRx.test(s) && endRx.test(s)) + s = s.replace(startRx, "").replace(endRx, "") + return JSON5TryParse(s) +} + diff --git a/packages/sample/genaisrc/data-analyst.genai.mjs b/packages/sample/genaisrc/data-analyst.genai.mjs index a3e2906960..f0de129e62 100644 --- a/packages/sample/genaisrc/data-analyst.genai.mjs +++ b/packages/sample/genaisrc/data-analyst.genai.mjs @@ -5,4 +5,7 @@ script({ const data = def("DATA", env.files, { sliceSample: 25 }) -$`Analyze ${data} with a detailed statistical analysis.` +$`Analyze ${data} with a detailed statistical analysis. + +- Do not generate visualizations. +- Do not return python code.` From 4dabebd474fd1ac6a5c92bf961428a8e72c67ce5 Mon Sep 17 00:00:00 2001 From: pelikhan Date: Tue, 13 Aug 2024 08:49:44 -0700 Subject: [PATCH 09/16] pandas sample --- docs/genaisrc/genaiscript.d.ts | 2 +- docs/src/components/BuiltinTools.mdx | 2 +- docs/src/content/docs/reference/scripts/system.mdx | 11 ++++++----- genaisrc/genaiscript.d.ts | 2 +- packages/cli/src/docker.ts | 6 +++++- packages/cli/src/run.ts | 2 +- packages/core/src/chat.ts | 9 +++++---- packages/core/src/expander.ts | 12 ++++++------ packages/core/src/genaisrc/genaiscript.d.ts | 2 +- .../src/genaisrc/system.python_interpreter.genai.js | 10 ++++++---- packages/core/src/promptrunner.ts | 2 +- packages/core/src/util.ts | 2 +- packages/sample/genaisrc/data-analyst.genai.mjs | 3 ++- packages/sample/genaisrc/genaiscript.d.ts | 2 +- packages/sample/genaisrc/node/genaiscript.d.ts | 2 +- packages/sample/genaisrc/pandas.genai.mjs | 7 +++++++ packages/sample/genaisrc/python/genaiscript.d.ts | 2 +- packages/sample/genaisrc/style/genaiscript.d.ts | 2 +- packages/sample/src/aici/genaiscript.d.ts | 2 +- packages/sample/src/errors/genaiscript.d.ts | 2 +- packages/sample/src/makecode/genaiscript.d.ts | 2 +- packages/sample/src/tla/genaiscript.d.ts | 2 +- packages/sample/src/vision/genaiscript.d.ts | 2 +- slides/genaisrc/genaiscript.d.ts | 2 +- 24 files changed, 54 insertions(+), 38 deletions(-) create mode 100644 packages/sample/genaisrc/pandas.genai.mjs diff --git a/docs/genaisrc/genaiscript.d.ts b/docs/genaisrc/genaiscript.d.ts index e618a36957..01a13b6322 100644 --- a/docs/genaisrc/genaiscript.d.ts +++ b/docs/genaisrc/genaiscript.d.ts @@ -231,7 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression -* - `python_interpreter`: Executes python code in a docker container. The process output is returned. +* - `python_interpreter`: Executes python code in a docker container. The process output is returned. Use 'print' to output data. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. diff --git a/docs/src/components/BuiltinTools.mdx b/docs/src/components/BuiltinTools.mdx index 0b3204b06e..a1c60457b5 100644 --- a/docs/src/components/BuiltinTools.mdx +++ b/docs/src/components/BuiltinTools.mdx @@ -10,7 +10,7 @@ import { LinkCard } from '@astrojs/starlight/components'; - + diff --git a/docs/src/content/docs/reference/scripts/system.mdx b/docs/src/content/docs/reference/scripts/system.mdx index f9c24634d3..7241d374a0 100644 --- a/docs/src/content/docs/reference/scripts/system.mdx +++ b/docs/src/content/docs/reference/scripts/system.mdx @@ -596,7 +596,7 @@ Python Dockerized code execution -- tool `python_interpreter`: Executes python code in a docker container. The process output is returned. +- tool `python_interpreter`: Executes python code in a docker container. The process output is returned. Use 'print' to output data. `````js wrap title="system.python_interpreter" system({ @@ -609,13 +609,14 @@ let container = null defTool( "python_interpreter", - "Executes python code in a docker container. The process output is returned.", + "Executes python code in a docker container. The process output is returned. Use 'print' to output data.", { type: "object", properties: { requirements: { type: "string", - description: "pip install requirements.txt file to install", + description: + "list of packages to install using pip. should be using the pip install format.", }, main: { type: "string", @@ -627,9 +628,9 @@ defTool( async (args) => { const { requirements, main = "" } = args console.log(`python: running code...`) - container = await host.container({ image }) + container = await host.container({ image, networkEnabled: true }) if (requirements) { - console.log(`installing: ` + requirements.replace(/\n/g, ", ")) + console.log(`installing: ` + requirements) await container.writeText("requirements.txt", requirements) await container.exec("pip", [ "install", diff --git a/genaisrc/genaiscript.d.ts b/genaisrc/genaiscript.d.ts index e618a36957..01a13b6322 100644 --- a/genaisrc/genaiscript.d.ts +++ b/genaisrc/genaiscript.d.ts @@ -231,7 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression -* - `python_interpreter`: Executes python code in a docker container. The process output is returned. +* - `python_interpreter`: Executes python code in a docker container. The process output is returned. Use 'print' to output data. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. diff --git a/packages/cli/src/docker.ts b/packages/cli/src/docker.ts index d1e15183c7..5513139922 100644 --- a/packages/cli/src/docker.ts +++ b/packages/cli/src/docker.ts @@ -208,6 +208,10 @@ export class DockerManager { trace?.itemValue(`container`, container.id) trace?.itemValue(`cwd`, cwd) trace?.item(`\`${command}\` ${args.join(" ")}`) + if (!isQuiet) + logVerbose( + `container exec: ${command} ${args.join(" ")}` + ) let inspection = await container.inspect() trace?.itemValue( @@ -256,7 +260,7 @@ export class DockerManager { } if (sres.stderr) { trace?.detailsFenced(`stderr`, sres.stderr) - if (!isQuiet) logVerbose(sres.stdout) + if (!isQuiet) logVerbose(sres.stderr) } return sres diff --git a/packages/cli/src/run.ts b/packages/cli/src/run.ts index d894d65cbf..6a98e15d4e 100644 --- a/packages/cli/src/run.ts +++ b/packages/cli/src/run.ts @@ -124,7 +124,7 @@ export async function runScript( const ffs = await host.findFiles(arg, { applyGitIgnore: excludeGitIgnore, }) - if (!ffs.length) { + if (!ffs?.length) { return fail( `no files matching ${arg}`, FILES_NOT_FOUND_ERROR_CODE diff --git a/packages/core/src/chat.ts b/packages/core/src/chat.ts index fab53c91fb..4f41af8334 100644 --- a/packages/core/src/chat.ts +++ b/packages/core/src/chat.ts @@ -513,10 +513,11 @@ export async function executeChatSession( infoCb?.({ text: `prompting ${model} (~${estimateChatTokens(model, messages)} tokens)`, }) - trace.details( - `💬 messages (${messages.length})`, - renderMessagesToMarkdown(messages) - ) + if (messages) + trace.details( + `💬 messages (${messages.length})`, + renderMessagesToMarkdown(messages) + ) // make request let resp: ChatCompletionResponse diff --git a/packages/core/src/expander.ts b/packages/core/src/expander.ts index 92a84fcf2f..1e8438cf20 100644 --- a/packages/core/src/expander.ts +++ b/packages/core/src/expander.ts @@ -164,6 +164,7 @@ export async function expandTemplate( ) { const model = options.model assert(!!model) + const messages: ChatCompletionMessageParam[] = [] const cancellationToken = options.cancellationToken const systems = resolveSystems(prj, template) const systemTemplates = systems.map((s) => prj.getTemplate(s)) @@ -189,7 +190,7 @@ export async function expandTemplate( normalizeInt(env.vars["maxToolCalls"]) ?? normalizeInt(env.vars["max_tool_calls"]) ?? template.maxToolCalls ?? - MAX_TOOL_CALLS + MAX_TOOL_CALLS let seed = options.seed ?? normalizeInt(env.vars["seed"]) ?? template.seed if (seed !== undefined) seed = seed >> 0 @@ -222,23 +223,22 @@ export async function expandTemplate( if (prompt.status !== "success" || prompt.text === "") // cancelled - return { status: prompt.status, statusText: prompt.statusText } + return { status: prompt.status, statusText: prompt.statusText, messages } if (cancellationToken?.isCancellationRequested) - return { status: "cancelled", statusText: "user cancelled" } + return { status: "cancelled", statusText: "user cancelled", messages } const systemMessage: ChatCompletionSystemMessageParam = { role: "system", content: "", } - const messages: ChatCompletionMessageParam[] = [] if (prompt.text) messages.push(toChatCompletionUserMessage(prompt.text, prompt.images)) if (prompt.aici) messages.push(prompt.aici) for (let i = 0; i < systems.length; ++i) { if (cancellationToken?.isCancellationRequested) - return { status: "cancelled", statusText: "user cancelled" } + return { status: "cancelled", statusText: "user cancelled", messages } let systemTemplate = systems[i] let system = prj.getTemplate(systemTemplate) @@ -277,7 +277,7 @@ export async function expandTemplate( trace.endDetails() if (sysr.status !== "success") - return { status: sysr.status, statusText: sysr.statusText } + return { status: sysr.status, statusText: sysr.statusText, messages } } const responseSchema = promptParametersSchemaToJSONSchema( diff --git a/packages/core/src/genaisrc/genaiscript.d.ts b/packages/core/src/genaisrc/genaiscript.d.ts index e618a36957..01a13b6322 100644 --- a/packages/core/src/genaisrc/genaiscript.d.ts +++ b/packages/core/src/genaisrc/genaiscript.d.ts @@ -231,7 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression -* - `python_interpreter`: Executes python code in a docker container. The process output is returned. +* - `python_interpreter`: Executes python code in a docker container. The process output is returned. Use 'print' to output data. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. diff --git a/packages/core/src/genaisrc/system.python_interpreter.genai.js b/packages/core/src/genaisrc/system.python_interpreter.genai.js index 540ad5878f..1053634070 100644 --- a/packages/core/src/genaisrc/system.python_interpreter.genai.js +++ b/packages/core/src/genaisrc/system.python_interpreter.genai.js @@ -8,13 +8,14 @@ let container = null defTool( "python_interpreter", - "Executes python code in a docker container. The process output is returned.", + "Executes python code in a docker container. The process output is returned. Use 'print' to output data.", { type: "object", properties: { requirements: { type: "string", - description: "pip install requirements.txt file to install", + description: + "list of packages to install using pip. should be using the pip install format.", }, main: { type: "string", @@ -26,13 +27,14 @@ defTool( async (args) => { const { requirements, main = "" } = args console.log(`python: running code...`) - container = await host.container({ image }) + container = await host.container({ image, networkEnabled: true }) if (requirements) { - console.log(`installing: ` + requirements.replace(/\n/g, ", ")) + console.log(`installing: ` + requirements) await container.writeText("requirements.txt", requirements) await container.exec("pip", [ "install", "--no-cache-dir", + "--root-user-action", "ignore", "-r", "requirements.txt", ]) diff --git a/packages/core/src/promptrunner.ts b/packages/core/src/promptrunner.ts index 67208fef9b..6293e4a1d4 100644 --- a/packages/core/src/promptrunner.ts +++ b/packages/core/src/promptrunner.ts @@ -119,7 +119,7 @@ export async function runTemplate( ) // if the expansion failed, show the user the trace - if (status !== "success") { + if (status !== "success" || !messages.length) { trace.renderErrors() return { status, diff --git a/packages/core/src/util.ts b/packages/core/src/util.ts index 43ec7fc7db..ef81bf25dd 100644 --- a/packages/core/src/util.ts +++ b/packages/core/src/util.ts @@ -180,10 +180,10 @@ export function logWarn(msg: string) { export function logError(msg: string | Error | SerializedError) { const { message, ...e } = serializeError(msg) if (message) host.log(LogLevel.Error, message) + console.debug(msg) const se = YAMLStringify(e) if (!/^\s*\{\}\s*$/) host.log(LogLevel.Info, se) } - export function concatArrays(...arrays: T[][]): T[] { if (arrays.length == 0) return [] return arrays[0].concat(...arrays.slice(1)) diff --git a/packages/sample/genaisrc/data-analyst.genai.mjs b/packages/sample/genaisrc/data-analyst.genai.mjs index f0de129e62..93a9b24ccd 100644 --- a/packages/sample/genaisrc/data-analyst.genai.mjs +++ b/packages/sample/genaisrc/data-analyst.genai.mjs @@ -8,4 +8,5 @@ const data = def("DATA", env.files, { sliceSample: 25 }) $`Analyze ${data} with a detailed statistical analysis. - Do not generate visualizations. -- Do not return python code.` +- Validate computations with code. +` diff --git a/packages/sample/genaisrc/genaiscript.d.ts b/packages/sample/genaisrc/genaiscript.d.ts index e618a36957..01a13b6322 100644 --- a/packages/sample/genaisrc/genaiscript.d.ts +++ b/packages/sample/genaisrc/genaiscript.d.ts @@ -231,7 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression -* - `python_interpreter`: Executes python code in a docker container. The process output is returned. +* - `python_interpreter`: Executes python code in a docker container. The process output is returned. Use 'print' to output data. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. diff --git a/packages/sample/genaisrc/node/genaiscript.d.ts b/packages/sample/genaisrc/node/genaiscript.d.ts index e618a36957..01a13b6322 100644 --- a/packages/sample/genaisrc/node/genaiscript.d.ts +++ b/packages/sample/genaisrc/node/genaiscript.d.ts @@ -231,7 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression -* - `python_interpreter`: Executes python code in a docker container. The process output is returned. +* - `python_interpreter`: Executes python code in a docker container. The process output is returned. Use 'print' to output data. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. diff --git a/packages/sample/genaisrc/pandas.genai.mjs b/packages/sample/genaisrc/pandas.genai.mjs new file mode 100644 index 0000000000..f4769196a2 --- /dev/null +++ b/packages/sample/genaisrc/pandas.genai.mjs @@ -0,0 +1,7 @@ +const container = await host.container({ image: "python:3", networkEnabled: true }) +await container.exec("pip", ["install", "--no-cache-dir", "--root-user-action", "ignore", "pandas"]) +await container.writeText("main.py", `import pandas as pd +print(pd) +`) +const res = await container.exec('python', ["main.py"]) +console.log(YAML.stringify(res)) \ No newline at end of file diff --git a/packages/sample/genaisrc/python/genaiscript.d.ts b/packages/sample/genaisrc/python/genaiscript.d.ts index e618a36957..01a13b6322 100644 --- a/packages/sample/genaisrc/python/genaiscript.d.ts +++ b/packages/sample/genaisrc/python/genaiscript.d.ts @@ -231,7 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression -* - `python_interpreter`: Executes python code in a docker container. The process output is returned. +* - `python_interpreter`: Executes python code in a docker container. The process output is returned. Use 'print' to output data. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. diff --git a/packages/sample/genaisrc/style/genaiscript.d.ts b/packages/sample/genaisrc/style/genaiscript.d.ts index e618a36957..01a13b6322 100644 --- a/packages/sample/genaisrc/style/genaiscript.d.ts +++ b/packages/sample/genaisrc/style/genaiscript.d.ts @@ -231,7 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression -* - `python_interpreter`: Executes python code in a docker container. The process output is returned. +* - `python_interpreter`: Executes python code in a docker container. The process output is returned. Use 'print' to output data. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. diff --git a/packages/sample/src/aici/genaiscript.d.ts b/packages/sample/src/aici/genaiscript.d.ts index e618a36957..01a13b6322 100644 --- a/packages/sample/src/aici/genaiscript.d.ts +++ b/packages/sample/src/aici/genaiscript.d.ts @@ -231,7 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression -* - `python_interpreter`: Executes python code in a docker container. The process output is returned. +* - `python_interpreter`: Executes python code in a docker container. The process output is returned. Use 'print' to output data. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. diff --git a/packages/sample/src/errors/genaiscript.d.ts b/packages/sample/src/errors/genaiscript.d.ts index e618a36957..01a13b6322 100644 --- a/packages/sample/src/errors/genaiscript.d.ts +++ b/packages/sample/src/errors/genaiscript.d.ts @@ -231,7 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression -* - `python_interpreter`: Executes python code in a docker container. The process output is returned. +* - `python_interpreter`: Executes python code in a docker container. The process output is returned. Use 'print' to output data. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. diff --git a/packages/sample/src/makecode/genaiscript.d.ts b/packages/sample/src/makecode/genaiscript.d.ts index e618a36957..01a13b6322 100644 --- a/packages/sample/src/makecode/genaiscript.d.ts +++ b/packages/sample/src/makecode/genaiscript.d.ts @@ -231,7 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression -* - `python_interpreter`: Executes python code in a docker container. The process output is returned. +* - `python_interpreter`: Executes python code in a docker container. The process output is returned. Use 'print' to output data. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. diff --git a/packages/sample/src/tla/genaiscript.d.ts b/packages/sample/src/tla/genaiscript.d.ts index e618a36957..01a13b6322 100644 --- a/packages/sample/src/tla/genaiscript.d.ts +++ b/packages/sample/src/tla/genaiscript.d.ts @@ -231,7 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression -* - `python_interpreter`: Executes python code in a docker container. The process output is returned. +* - `python_interpreter`: Executes python code in a docker container. The process output is returned. Use 'print' to output data. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. diff --git a/packages/sample/src/vision/genaiscript.d.ts b/packages/sample/src/vision/genaiscript.d.ts index e618a36957..01a13b6322 100644 --- a/packages/sample/src/vision/genaiscript.d.ts +++ b/packages/sample/src/vision/genaiscript.d.ts @@ -231,7 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression -* - `python_interpreter`: Executes python code in a docker container. The process output is returned. +* - `python_interpreter`: Executes python code in a docker container. The process output is returned. Use 'print' to output data. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. diff --git a/slides/genaisrc/genaiscript.d.ts b/slides/genaisrc/genaiscript.d.ts index e618a36957..01a13b6322 100644 --- a/slides/genaisrc/genaiscript.d.ts +++ b/slides/genaisrc/genaiscript.d.ts @@ -231,7 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression -* - `python_interpreter`: Executes python code in a docker container. The process output is returned. +* - `python_interpreter`: Executes python code in a docker container. The process output is returned. Use 'print' to output data. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. From 4bf633123f45c141437cb22735b46b9d9878b7de Mon Sep 17 00:00:00 2001 From: pelikhan Date: Tue, 13 Aug 2024 08:50:00 -0700 Subject: [PATCH 10/16] formatting --- packages/core/src/genaisrc/system.python_interpreter.genai.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/core/src/genaisrc/system.python_interpreter.genai.js b/packages/core/src/genaisrc/system.python_interpreter.genai.js index 1053634070..0f396c2001 100644 --- a/packages/core/src/genaisrc/system.python_interpreter.genai.js +++ b/packages/core/src/genaisrc/system.python_interpreter.genai.js @@ -34,7 +34,8 @@ defTool( await container.exec("pip", [ "install", "--no-cache-dir", - "--root-user-action", "ignore", + "--root-user-action", + "ignore", "-r", "requirements.txt", ]) From 4561569fddcd75827d06a0d01fa5e479c2190f51 Mon Sep 17 00:00:00 2001 From: pelikhan Date: Tue, 13 Aug 2024 09:00:56 -0700 Subject: [PATCH 11/16] escape requirements format --- docs/src/content/docs/reference/scripts/system.mdx | 9 +++++++-- .../core/src/genaisrc/system.python_interpreter.genai.js | 8 +++++--- packages/sample/genaisrc/pandas.genai.mjs | 2 +- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/docs/src/content/docs/reference/scripts/system.mdx b/docs/src/content/docs/reference/scripts/system.mdx index 7241d374a0..9dfe008a21 100644 --- a/docs/src/content/docs/reference/scripts/system.mdx +++ b/docs/src/content/docs/reference/scripts/system.mdx @@ -616,7 +616,10 @@ defTool( requirements: { type: "string", description: - "list of packages to install using pip. should be using the pip install format.", + `list of packages and versions to install using pip. should be using the pip install format: +=== +=== +` }, main: { type: "string", @@ -631,10 +634,12 @@ defTool( container = await host.container({ image, networkEnabled: true }) if (requirements) { console.log(`installing: ` + requirements) - await container.writeText("requirements.txt", requirements) + await container.writeText("requirements.txt", requirements.replace(/[ ,]\s*/g, "\n")) await container.exec("pip", [ "install", "--no-cache-dir", + "--root-user-action", + "ignore", "-r", "requirements.txt", ]) diff --git a/packages/core/src/genaisrc/system.python_interpreter.genai.js b/packages/core/src/genaisrc/system.python_interpreter.genai.js index 0f396c2001..3038ec60c5 100644 --- a/packages/core/src/genaisrc/system.python_interpreter.genai.js +++ b/packages/core/src/genaisrc/system.python_interpreter.genai.js @@ -15,7 +15,10 @@ defTool( requirements: { type: "string", description: - "list of packages to install using pip. should be using the pip install format.", + `list of packages and versions to install using pip. should be using the pip install format: +=== +=== +` }, main: { type: "string", @@ -30,10 +33,9 @@ defTool( container = await host.container({ image, networkEnabled: true }) if (requirements) { console.log(`installing: ` + requirements) - await container.writeText("requirements.txt", requirements) + await container.writeText("requirements.txt", requirements.replace(/[ ,]\s*/g, "\n")) await container.exec("pip", [ "install", - "--no-cache-dir", "--root-user-action", "ignore", "-r", diff --git a/packages/sample/genaisrc/pandas.genai.mjs b/packages/sample/genaisrc/pandas.genai.mjs index f4769196a2..4939e090f6 100644 --- a/packages/sample/genaisrc/pandas.genai.mjs +++ b/packages/sample/genaisrc/pandas.genai.mjs @@ -1,5 +1,5 @@ const container = await host.container({ image: "python:3", networkEnabled: true }) -await container.exec("pip", ["install", "--no-cache-dir", "--root-user-action", "ignore", "pandas"]) +await container.exec("pip", ["install", "--root-user-action", "ignore", "pandas"]) await container.writeText("main.py", `import pandas as pd print(pd) `) From d98d3e747bdb332f5097a629dd7a4292eaf2fa1a Mon Sep 17 00:00:00 2001 From: pelikhan Date: Tue, 13 Aug 2024 09:01:59 -0700 Subject: [PATCH 12/16] improving description --- .../core/src/genaisrc/system.python_interpreter.genai.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/core/src/genaisrc/system.python_interpreter.genai.js b/packages/core/src/genaisrc/system.python_interpreter.genai.js index 3038ec60c5..4d45071933 100644 --- a/packages/core/src/genaisrc/system.python_interpreter.genai.js +++ b/packages/core/src/genaisrc/system.python_interpreter.genai.js @@ -8,21 +8,21 @@ let container = null defTool( "python_interpreter", - "Executes python code in a docker container. The process output is returned. Use 'print' to output data.", + "Executes python 3 code in a docker container. The process output is returned. Use 'print' to output data.", { type: "object", properties: { requirements: { type: "string", description: - `list of packages and versions to install using pip. should be using the pip install format: + `list of pip packages and versions to install using pip. should be using the pip install format: === === ` }, main: { type: "string", - description: "python source code to execute", + description: "python 3 source code to execute", }, }, required: ["requirements", "main"], From e259dd6feaf52284407160a0f4e29747a6ed795a Mon Sep 17 00:00:00 2001 From: pelikhan Date: Tue, 13 Aug 2024 09:13:52 -0700 Subject: [PATCH 13/16] simplify requirements format --- docs/genaisrc/genaiscript.d.ts | 2 +- docs/src/components/BuiltinTools.mdx | 2 +- .../content/docs/reference/scripts/system.mdx | 26 ++++++++++--------- genaisrc/genaiscript.d.ts | 2 +- packages/core/src/genaisrc/genaiscript.d.ts | 2 +- .../system.python_interpreter.genai.js | 23 +++++++++------- packages/sample/genaisrc/genaiscript.d.ts | 2 +- .../sample/genaisrc/node/genaiscript.d.ts | 2 +- .../sample/genaisrc/python/genaiscript.d.ts | 2 +- .../sample/genaisrc/style/genaiscript.d.ts | 2 +- packages/sample/src/aici/genaiscript.d.ts | 2 +- packages/sample/src/errors/genaiscript.d.ts | 2 +- packages/sample/src/makecode/genaiscript.d.ts | 2 +- packages/sample/src/tla/genaiscript.d.ts | 2 +- packages/sample/src/vision/genaiscript.d.ts | 2 +- slides/genaisrc/genaiscript.d.ts | 2 +- 16 files changed, 41 insertions(+), 36 deletions(-) diff --git a/docs/genaisrc/genaiscript.d.ts b/docs/genaisrc/genaiscript.d.ts index 01a13b6322..690b174bb3 100644 --- a/docs/genaisrc/genaiscript.d.ts +++ b/docs/genaisrc/genaiscript.d.ts @@ -231,7 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression -* - `python_interpreter`: Executes python code in a docker container. The process output is returned. Use 'print' to output data. +* - `python_interpreter`: Executes python 3.12 code in a docker container. The process output is returned. Use 'print' to output data. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. diff --git a/docs/src/components/BuiltinTools.mdx b/docs/src/components/BuiltinTools.mdx index a1c60457b5..83dec7df0f 100644 --- a/docs/src/components/BuiltinTools.mdx +++ b/docs/src/components/BuiltinTools.mdx @@ -10,7 +10,7 @@ import { LinkCard } from '@astrojs/starlight/components'; - + diff --git a/docs/src/content/docs/reference/scripts/system.mdx b/docs/src/content/docs/reference/scripts/system.mdx index 9dfe008a21..f97f59743b 100644 --- a/docs/src/content/docs/reference/scripts/system.mdx +++ b/docs/src/content/docs/reference/scripts/system.mdx @@ -596,34 +596,33 @@ Python Dockerized code execution -- tool `python_interpreter`: Executes python code in a docker container. The process output is returned. Use 'print' to output data. +- tool `python_interpreter`: Executes python 3.12 code in a docker container. The process output is returned. Use 'print' to output data. `````js wrap title="system.python_interpreter" system({ title: "Python Dockerized code execution", }) -const image = env.vars.pythonImage ?? "python:3" +const image = env.vars.pythonImage ?? "python:3.12" let container = null defTool( "python_interpreter", - "Executes python code in a docker container. The process output is returned. Use 'print' to output data.", + "Executes python 3.12 code in a docker container. The process output is returned. Use 'print' to output data.", { type: "object", properties: { requirements: { type: "string", - description: - `list of packages and versions to install using pip. should be using the pip install format: -=== -=== -` + description: `list of pip packages to install using pip. should be using the pip install format: + + +`, }, main: { type: "string", - description: "python source code to execute", + description: "python 3.12 source code to execute", }, }, required: ["requirements", "main"], @@ -634,15 +633,18 @@ defTool( container = await host.container({ image, networkEnabled: true }) if (requirements) { console.log(`installing: ` + requirements) - await container.writeText("requirements.txt", requirements.replace(/[ ,]\s*/g, "\n")) - await container.exec("pip", [ + await container.writeText( + "requirements.txt", + requirements.replace(/[ ,]\s*/g, "\n") + ) + const res = await container.exec("pip", [ "install", - "--no-cache-dir", "--root-user-action", "ignore", "-r", "requirements.txt", ]) + if (res.failed) throw new Error(`Failed to install requirements`) } console.log(`code: ` + main) diff --git a/genaisrc/genaiscript.d.ts b/genaisrc/genaiscript.d.ts index 01a13b6322..690b174bb3 100644 --- a/genaisrc/genaiscript.d.ts +++ b/genaisrc/genaiscript.d.ts @@ -231,7 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression -* - `python_interpreter`: Executes python code in a docker container. The process output is returned. Use 'print' to output data. +* - `python_interpreter`: Executes python 3.12 code in a docker container. The process output is returned. Use 'print' to output data. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. diff --git a/packages/core/src/genaisrc/genaiscript.d.ts b/packages/core/src/genaisrc/genaiscript.d.ts index 01a13b6322..690b174bb3 100644 --- a/packages/core/src/genaisrc/genaiscript.d.ts +++ b/packages/core/src/genaisrc/genaiscript.d.ts @@ -231,7 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression -* - `python_interpreter`: Executes python code in a docker container. The process output is returned. Use 'print' to output data. +* - `python_interpreter`: Executes python 3.12 code in a docker container. The process output is returned. Use 'print' to output data. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. diff --git a/packages/core/src/genaisrc/system.python_interpreter.genai.js b/packages/core/src/genaisrc/system.python_interpreter.genai.js index 4d45071933..8e886116b2 100644 --- a/packages/core/src/genaisrc/system.python_interpreter.genai.js +++ b/packages/core/src/genaisrc/system.python_interpreter.genai.js @@ -2,27 +2,26 @@ system({ title: "Python Dockerized code execution", }) -const image = env.vars.pythonImage ?? "python:3" +const image = env.vars.pythonImage ?? "python:3.12" let container = null defTool( "python_interpreter", - "Executes python 3 code in a docker container. The process output is returned. Use 'print' to output data.", + "Executes python 3.12 code in a docker container. The process output is returned. Use 'print' to output data.", { type: "object", properties: { requirements: { type: "string", - description: - `list of pip packages and versions to install using pip. should be using the pip install format: -=== -=== -` + description: `list of pip packages to install using pip. should be using the pip install format: + + +`, }, main: { type: "string", - description: "python 3 source code to execute", + description: "python 3.12 source code to execute", }, }, required: ["requirements", "main"], @@ -33,14 +32,18 @@ defTool( container = await host.container({ image, networkEnabled: true }) if (requirements) { console.log(`installing: ` + requirements) - await container.writeText("requirements.txt", requirements.replace(/[ ,]\s*/g, "\n")) - await container.exec("pip", [ + await container.writeText( + "requirements.txt", + requirements.replace(/[ ,]\s*/g, "\n") + ) + const res = await container.exec("pip", [ "install", "--root-user-action", "ignore", "-r", "requirements.txt", ]) + if (res.failed) throw new Error(`Failed to install requirements`) } console.log(`code: ` + main) diff --git a/packages/sample/genaisrc/genaiscript.d.ts b/packages/sample/genaisrc/genaiscript.d.ts index 01a13b6322..690b174bb3 100644 --- a/packages/sample/genaisrc/genaiscript.d.ts +++ b/packages/sample/genaisrc/genaiscript.d.ts @@ -231,7 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression -* - `python_interpreter`: Executes python code in a docker container. The process output is returned. Use 'print' to output data. +* - `python_interpreter`: Executes python 3.12 code in a docker container. The process output is returned. Use 'print' to output data. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. diff --git a/packages/sample/genaisrc/node/genaiscript.d.ts b/packages/sample/genaisrc/node/genaiscript.d.ts index 01a13b6322..690b174bb3 100644 --- a/packages/sample/genaisrc/node/genaiscript.d.ts +++ b/packages/sample/genaisrc/node/genaiscript.d.ts @@ -231,7 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression -* - `python_interpreter`: Executes python code in a docker container. The process output is returned. Use 'print' to output data. +* - `python_interpreter`: Executes python 3.12 code in a docker container. The process output is returned. Use 'print' to output data. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. diff --git a/packages/sample/genaisrc/python/genaiscript.d.ts b/packages/sample/genaisrc/python/genaiscript.d.ts index 01a13b6322..690b174bb3 100644 --- a/packages/sample/genaisrc/python/genaiscript.d.ts +++ b/packages/sample/genaisrc/python/genaiscript.d.ts @@ -231,7 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression -* - `python_interpreter`: Executes python code in a docker container. The process output is returned. Use 'print' to output data. +* - `python_interpreter`: Executes python 3.12 code in a docker container. The process output is returned. Use 'print' to output data. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. diff --git a/packages/sample/genaisrc/style/genaiscript.d.ts b/packages/sample/genaisrc/style/genaiscript.d.ts index 01a13b6322..690b174bb3 100644 --- a/packages/sample/genaisrc/style/genaiscript.d.ts +++ b/packages/sample/genaisrc/style/genaiscript.d.ts @@ -231,7 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression -* - `python_interpreter`: Executes python code in a docker container. The process output is returned. Use 'print' to output data. +* - `python_interpreter`: Executes python 3.12 code in a docker container. The process output is returned. Use 'print' to output data. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. diff --git a/packages/sample/src/aici/genaiscript.d.ts b/packages/sample/src/aici/genaiscript.d.ts index 01a13b6322..690b174bb3 100644 --- a/packages/sample/src/aici/genaiscript.d.ts +++ b/packages/sample/src/aici/genaiscript.d.ts @@ -231,7 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression -* - `python_interpreter`: Executes python code in a docker container. The process output is returned. Use 'print' to output data. +* - `python_interpreter`: Executes python 3.12 code in a docker container. The process output is returned. Use 'print' to output data. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. diff --git a/packages/sample/src/errors/genaiscript.d.ts b/packages/sample/src/errors/genaiscript.d.ts index 01a13b6322..690b174bb3 100644 --- a/packages/sample/src/errors/genaiscript.d.ts +++ b/packages/sample/src/errors/genaiscript.d.ts @@ -231,7 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression -* - `python_interpreter`: Executes python code in a docker container. The process output is returned. Use 'print' to output data. +* - `python_interpreter`: Executes python 3.12 code in a docker container. The process output is returned. Use 'print' to output data. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. diff --git a/packages/sample/src/makecode/genaiscript.d.ts b/packages/sample/src/makecode/genaiscript.d.ts index 01a13b6322..690b174bb3 100644 --- a/packages/sample/src/makecode/genaiscript.d.ts +++ b/packages/sample/src/makecode/genaiscript.d.ts @@ -231,7 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression -* - `python_interpreter`: Executes python code in a docker container. The process output is returned. Use 'print' to output data. +* - `python_interpreter`: Executes python 3.12 code in a docker container. The process output is returned. Use 'print' to output data. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. diff --git a/packages/sample/src/tla/genaiscript.d.ts b/packages/sample/src/tla/genaiscript.d.ts index 01a13b6322..690b174bb3 100644 --- a/packages/sample/src/tla/genaiscript.d.ts +++ b/packages/sample/src/tla/genaiscript.d.ts @@ -231,7 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression -* - `python_interpreter`: Executes python code in a docker container. The process output is returned. Use 'print' to output data. +* - `python_interpreter`: Executes python 3.12 code in a docker container. The process output is returned. Use 'print' to output data. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. diff --git a/packages/sample/src/vision/genaiscript.d.ts b/packages/sample/src/vision/genaiscript.d.ts index 01a13b6322..690b174bb3 100644 --- a/packages/sample/src/vision/genaiscript.d.ts +++ b/packages/sample/src/vision/genaiscript.d.ts @@ -231,7 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression -* - `python_interpreter`: Executes python code in a docker container. The process output is returned. Use 'print' to output data. +* - `python_interpreter`: Executes python 3.12 code in a docker container. The process output is returned. Use 'print' to output data. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. diff --git a/slides/genaisrc/genaiscript.d.ts b/slides/genaisrc/genaiscript.d.ts index 01a13b6322..690b174bb3 100644 --- a/slides/genaisrc/genaiscript.d.ts +++ b/slides/genaisrc/genaiscript.d.ts @@ -231,7 +231,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression -* - `python_interpreter`: Executes python code in a docker container. The process output is returned. Use 'print' to output data. +* - `python_interpreter`: Executes python 3.12 code in a docker container. The process output is returned. Use 'print' to output data. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. From e67ef5ce7cd39660664195bb43fdce8fd6a8bae3 Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Tue, 13 Aug 2024 16:36:37 +0000 Subject: [PATCH 14/16] pand test --- packages/sample/genaisrc/pandas.genai.mjs | 24 +++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/packages/sample/genaisrc/pandas.genai.mjs b/packages/sample/genaisrc/pandas.genai.mjs index 4939e090f6..64d5ba6ee5 100644 --- a/packages/sample/genaisrc/pandas.genai.mjs +++ b/packages/sample/genaisrc/pandas.genai.mjs @@ -1,7 +1,19 @@ -const container = await host.container({ image: "python:3", networkEnabled: true }) -await container.exec("pip", ["install", "--root-user-action", "ignore", "pandas"]) -await container.writeText("main.py", `import pandas as pd +script({ tests: {} }) +const container = await host.container({ + image: "python:3.12", + networkEnabled: true, +}) +await container.exec("pip", [ + "install", + "--root-user-action", + "ignore", + "pandas", +]) +await container.writeText( + "main.py", + `import pandas as pd print(pd) -`) -const res = await container.exec('python', ["main.py"]) -console.log(YAML.stringify(res)) \ No newline at end of file +` +) +const res = await container.exec("python", ["main.py"]) +if (res.failed) throw new Error("import failed") From 8fbef3750004c39f46cf477da5adfc0b4bf23260 Mon Sep 17 00:00:00 2001 From: pelikhan Date: Tue, 13 Aug 2024 11:36:40 -0700 Subject: [PATCH 15/16] better intellisense for tools/system scripts --- docs/genaisrc/genaiscript.d.ts | 6 ++++-- genaisrc/genaiscript.d.ts | 6 ++++-- packages/core/src/genaisrc/genaiscript.d.ts | 6 ++++-- packages/core/src/scripts.ts | 12 ++++++------ packages/core/src/types/prompt_template.d.ts | 6 ++++-- packages/sample/genaisrc/genaiscript.d.ts | 6 ++++-- packages/sample/genaisrc/node/genaiscript.d.ts | 6 ++++-- packages/sample/genaisrc/python/genaiscript.d.ts | 6 ++++-- packages/sample/genaisrc/style/genaiscript.d.ts | 6 ++++-- packages/sample/src/aici/genaiscript.d.ts | 6 ++++-- packages/sample/src/errors/genaiscript.d.ts | 6 ++++-- packages/sample/src/makecode/genaiscript.d.ts | 6 ++++-- packages/sample/src/tla/genaiscript.d.ts | 6 ++++-- packages/sample/src/vision/genaiscript.d.ts | 6 ++++-- slides/genaisrc/genaiscript.d.ts | 6 ++++-- 15 files changed, 62 insertions(+), 34 deletions(-) diff --git a/docs/genaisrc/genaiscript.d.ts b/docs/genaisrc/genaiscript.d.ts index 690b174bb3..8fccf3c5fa 100644 --- a/docs/genaisrc/genaiscript.d.ts +++ b/docs/genaisrc/genaiscript.d.ts @@ -1,3 +1,5 @@ +type OptionsOrString = (string & {}) | TOptions + interface PromptGenerationConsole { log(...data: any[]): void warn(...data: any[]): void @@ -65,9 +67,9 @@ interface PromptLike extends PromptDefinition { text?: string } -type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" +type SystemPromptId = OptionsOrString<"system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot"> -type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" +type SystemToolId = OptionsOrString<"fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search"> type FileMergeHandler = ( filename: string, diff --git a/genaisrc/genaiscript.d.ts b/genaisrc/genaiscript.d.ts index 690b174bb3..8fccf3c5fa 100644 --- a/genaisrc/genaiscript.d.ts +++ b/genaisrc/genaiscript.d.ts @@ -1,3 +1,5 @@ +type OptionsOrString = (string & {}) | TOptions + interface PromptGenerationConsole { log(...data: any[]): void warn(...data: any[]): void @@ -65,9 +67,9 @@ interface PromptLike extends PromptDefinition { text?: string } -type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" +type SystemPromptId = OptionsOrString<"system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot"> -type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" +type SystemToolId = OptionsOrString<"fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search"> type FileMergeHandler = ( filename: string, diff --git a/packages/core/src/genaisrc/genaiscript.d.ts b/packages/core/src/genaisrc/genaiscript.d.ts index 690b174bb3..8fccf3c5fa 100644 --- a/packages/core/src/genaisrc/genaiscript.d.ts +++ b/packages/core/src/genaisrc/genaiscript.d.ts @@ -1,3 +1,5 @@ +type OptionsOrString = (string & {}) | TOptions + interface PromptGenerationConsole { log(...data: any[]): void warn(...data: any[]): void @@ -65,9 +67,9 @@ interface PromptLike extends PromptDefinition { text?: string } -type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" +type SystemPromptId = OptionsOrString<"system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot"> -type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" +type SystemToolId = OptionsOrString<"fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search"> type FileMergeHandler = ( filename: string, diff --git a/packages/core/src/scripts.ts b/packages/core/src/scripts.ts index 9bbb11248d..f6e749627c 100644 --- a/packages/core/src/scripts.ts +++ b/packages/core/src/scripts.ts @@ -49,11 +49,11 @@ export async function fixPromptDefinitions(project: Project) { // update the system prompt identifiers defContent = defContent .replace( - "type SystemPromptId = string", - `type SystemPromptId = ${systems + "type SystemPromptId = OptionsOrString", + `type SystemPromptId = OptionsOrString<${systems .sort((a, b) => a.id.localeCompare(b.id)) .map((s) => JSON.stringify(s.id)) - .join(" | ")}` + .join(" | ")}>` ) .replace( " system?: SystemPromptId[]", @@ -67,11 +67,11 @@ ${systems.map((s) => `* - \`${s.id}\`: ${s.title || s.description}`).join("\n")} // update the tool prompt identifiers defContent = defContent .replace( - "type SystemToolId = string", - `type SystemToolId = ${tools + "type SystemToolId = OptionsOrString", + `type SystemToolId = OptionsOrString<${tools .sort((a, b) => a.name.localeCompare(b.name)) .map((s) => JSON.stringify(s.name)) - .join(" | ")}` + .join(" | ")}>` ) .replace( " tools?: SystemToolId[]", diff --git a/packages/core/src/types/prompt_template.d.ts b/packages/core/src/types/prompt_template.d.ts index aad8e437d7..92973d4113 100644 --- a/packages/core/src/types/prompt_template.d.ts +++ b/packages/core/src/types/prompt_template.d.ts @@ -1,3 +1,5 @@ +type OptionsOrString = (string & {}) | TOptions + interface PromptGenerationConsole { log(...data: any[]): void warn(...data: any[]): void @@ -65,9 +67,9 @@ interface PromptLike extends PromptDefinition { text?: string } -type SystemPromptId = string +type SystemPromptId = OptionsOrString -type SystemToolId = string +type SystemToolId = OptionsOrString type FileMergeHandler = ( filename: string, diff --git a/packages/sample/genaisrc/genaiscript.d.ts b/packages/sample/genaisrc/genaiscript.d.ts index 690b174bb3..8fccf3c5fa 100644 --- a/packages/sample/genaisrc/genaiscript.d.ts +++ b/packages/sample/genaisrc/genaiscript.d.ts @@ -1,3 +1,5 @@ +type OptionsOrString = (string & {}) | TOptions + interface PromptGenerationConsole { log(...data: any[]): void warn(...data: any[]): void @@ -65,9 +67,9 @@ interface PromptLike extends PromptDefinition { text?: string } -type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" +type SystemPromptId = OptionsOrString<"system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot"> -type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" +type SystemToolId = OptionsOrString<"fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search"> type FileMergeHandler = ( filename: string, diff --git a/packages/sample/genaisrc/node/genaiscript.d.ts b/packages/sample/genaisrc/node/genaiscript.d.ts index 690b174bb3..8fccf3c5fa 100644 --- a/packages/sample/genaisrc/node/genaiscript.d.ts +++ b/packages/sample/genaisrc/node/genaiscript.d.ts @@ -1,3 +1,5 @@ +type OptionsOrString = (string & {}) | TOptions + interface PromptGenerationConsole { log(...data: any[]): void warn(...data: any[]): void @@ -65,9 +67,9 @@ interface PromptLike extends PromptDefinition { text?: string } -type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" +type SystemPromptId = OptionsOrString<"system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot"> -type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" +type SystemToolId = OptionsOrString<"fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search"> type FileMergeHandler = ( filename: string, diff --git a/packages/sample/genaisrc/python/genaiscript.d.ts b/packages/sample/genaisrc/python/genaiscript.d.ts index 690b174bb3..8fccf3c5fa 100644 --- a/packages/sample/genaisrc/python/genaiscript.d.ts +++ b/packages/sample/genaisrc/python/genaiscript.d.ts @@ -1,3 +1,5 @@ +type OptionsOrString = (string & {}) | TOptions + interface PromptGenerationConsole { log(...data: any[]): void warn(...data: any[]): void @@ -65,9 +67,9 @@ interface PromptLike extends PromptDefinition { text?: string } -type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" +type SystemPromptId = OptionsOrString<"system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot"> -type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" +type SystemToolId = OptionsOrString<"fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search"> type FileMergeHandler = ( filename: string, diff --git a/packages/sample/genaisrc/style/genaiscript.d.ts b/packages/sample/genaisrc/style/genaiscript.d.ts index 690b174bb3..8fccf3c5fa 100644 --- a/packages/sample/genaisrc/style/genaiscript.d.ts +++ b/packages/sample/genaisrc/style/genaiscript.d.ts @@ -1,3 +1,5 @@ +type OptionsOrString = (string & {}) | TOptions + interface PromptGenerationConsole { log(...data: any[]): void warn(...data: any[]): void @@ -65,9 +67,9 @@ interface PromptLike extends PromptDefinition { text?: string } -type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" +type SystemPromptId = OptionsOrString<"system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot"> -type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" +type SystemToolId = OptionsOrString<"fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search"> type FileMergeHandler = ( filename: string, diff --git a/packages/sample/src/aici/genaiscript.d.ts b/packages/sample/src/aici/genaiscript.d.ts index 690b174bb3..8fccf3c5fa 100644 --- a/packages/sample/src/aici/genaiscript.d.ts +++ b/packages/sample/src/aici/genaiscript.d.ts @@ -1,3 +1,5 @@ +type OptionsOrString = (string & {}) | TOptions + interface PromptGenerationConsole { log(...data: any[]): void warn(...data: any[]): void @@ -65,9 +67,9 @@ interface PromptLike extends PromptDefinition { text?: string } -type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" +type SystemPromptId = OptionsOrString<"system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot"> -type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" +type SystemToolId = OptionsOrString<"fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search"> type FileMergeHandler = ( filename: string, diff --git a/packages/sample/src/errors/genaiscript.d.ts b/packages/sample/src/errors/genaiscript.d.ts index 690b174bb3..8fccf3c5fa 100644 --- a/packages/sample/src/errors/genaiscript.d.ts +++ b/packages/sample/src/errors/genaiscript.d.ts @@ -1,3 +1,5 @@ +type OptionsOrString = (string & {}) | TOptions + interface PromptGenerationConsole { log(...data: any[]): void warn(...data: any[]): void @@ -65,9 +67,9 @@ interface PromptLike extends PromptDefinition { text?: string } -type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" +type SystemPromptId = OptionsOrString<"system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot"> -type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" +type SystemToolId = OptionsOrString<"fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search"> type FileMergeHandler = ( filename: string, diff --git a/packages/sample/src/makecode/genaiscript.d.ts b/packages/sample/src/makecode/genaiscript.d.ts index 690b174bb3..8fccf3c5fa 100644 --- a/packages/sample/src/makecode/genaiscript.d.ts +++ b/packages/sample/src/makecode/genaiscript.d.ts @@ -1,3 +1,5 @@ +type OptionsOrString = (string & {}) | TOptions + interface PromptGenerationConsole { log(...data: any[]): void warn(...data: any[]): void @@ -65,9 +67,9 @@ interface PromptLike extends PromptDefinition { text?: string } -type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" +type SystemPromptId = OptionsOrString<"system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot"> -type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" +type SystemToolId = OptionsOrString<"fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search"> type FileMergeHandler = ( filename: string, diff --git a/packages/sample/src/tla/genaiscript.d.ts b/packages/sample/src/tla/genaiscript.d.ts index 690b174bb3..8fccf3c5fa 100644 --- a/packages/sample/src/tla/genaiscript.d.ts +++ b/packages/sample/src/tla/genaiscript.d.ts @@ -1,3 +1,5 @@ +type OptionsOrString = (string & {}) | TOptions + interface PromptGenerationConsole { log(...data: any[]): void warn(...data: any[]): void @@ -65,9 +67,9 @@ interface PromptLike extends PromptDefinition { text?: string } -type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" +type SystemPromptId = OptionsOrString<"system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot"> -type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" +type SystemToolId = OptionsOrString<"fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search"> type FileMergeHandler = ( filename: string, diff --git a/packages/sample/src/vision/genaiscript.d.ts b/packages/sample/src/vision/genaiscript.d.ts index 690b174bb3..8fccf3c5fa 100644 --- a/packages/sample/src/vision/genaiscript.d.ts +++ b/packages/sample/src/vision/genaiscript.d.ts @@ -1,3 +1,5 @@ +type OptionsOrString = (string & {}) | TOptions + interface PromptGenerationConsole { log(...data: any[]): void warn(...data: any[]): void @@ -65,9 +67,9 @@ interface PromptLike extends PromptDefinition { text?: string } -type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" +type SystemPromptId = OptionsOrString<"system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot"> -type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" +type SystemToolId = OptionsOrString<"fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search"> type FileMergeHandler = ( filename: string, diff --git a/slides/genaisrc/genaiscript.d.ts b/slides/genaisrc/genaiscript.d.ts index 690b174bb3..8fccf3c5fa 100644 --- a/slides/genaisrc/genaiscript.d.ts +++ b/slides/genaisrc/genaiscript.d.ts @@ -1,3 +1,5 @@ +type OptionsOrString = (string & {}) | TOptions + interface PromptGenerationConsole { log(...data: any[]): void warn(...data: any[]): void @@ -65,9 +67,9 @@ interface PromptLike extends PromptDefinition { text?: string } -type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" +type SystemPromptId = OptionsOrString<"system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot"> -type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" +type SystemToolId = OptionsOrString<"fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search"> type FileMergeHandler = ( filename: string, From 1a8f515d5fcc1db8594018dcc47e3c9d2e6f21ee Mon Sep 17 00:00:00 2001 From: pelikhan Date: Tue, 13 Aug 2024 11:51:04 -0700 Subject: [PATCH 16/16] format stdout/err in logs --- packages/cli/src/docker.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/docker.ts b/packages/cli/src/docker.ts index 5513139922..af3885f5d3 100644 --- a/packages/cli/src/docker.ts +++ b/packages/cli/src/docker.ts @@ -255,11 +255,11 @@ export class DockerManager { `exit code: ${sres.exitCode}` ) if (sres.stdout) { - trace?.detailsFenced(`stdout`, sres.stdout) + trace?.detailsFenced(`stdout`, sres.stdout, "txt") if (!isQuiet) logVerbose(sres.stdout) } if (sres.stderr) { - trace?.detailsFenced(`stderr`, sres.stderr) + trace?.detailsFenced(`stderr`, sres.stderr, "txt") if (!isQuiet) logVerbose(sres.stderr) }