diff --git a/docs/genaisrc/genaiscript.d.ts b/docs/genaisrc/genaiscript.d.ts index 6e01d272c5..8fccf3c5fa 100644 --- a/docs/genaisrc/genaiscript.d.ts +++ b/docs/genaisrc/genaiscript.d.ts @@ -1,3 +1,5 @@ +type OptionsOrString = (string & {}) | TOptions + interface PromptGenerationConsole { log(...data: any[]): void warn(...data: any[]): void @@ -65,9 +67,9 @@ interface PromptLike extends PromptDefinition { text?: string } -type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" +type SystemPromptId = OptionsOrString<"system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot"> -type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" +type SystemToolId = OptionsOrString<"fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search"> type FileMergeHandler = ( filename: string, @@ -210,6 +212,7 @@ interface ScriptRuntimeOptions { * - `system.functions`: use functions * - `system.math`: Math expression evaluator * - `system.python`: Expert at generating and understanding Python code. +* - `system.python_interpreter`: Python Dockerized code execution * - `system.retrieval_fuzz_search`: Full Text Fuzzy Search * - `system.retrieval_vector_search`: Embeddings Vector Search * - `system.retrieval_web_search`: Web Search @@ -230,6 +233,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression +* - `python_interpreter`: Executes python 3.12 code in a docker container. The process output is returned. Use 'print' to output data. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. @@ -1622,6 +1626,11 @@ interface ContainerHost extends ShellHost { * @param toContainer directory in the container */ copyTo(fromHost: string | string[], toContainer: string): Promise + + /** + * Stops and cleans out the container + */ + stop(): Promise } interface PromptContext extends ChatGenerationContext { diff --git a/docs/src/components/BuiltinTools.mdx b/docs/src/components/BuiltinTools.mdx index acd6b4c9fc..83dec7df0f 100644 --- a/docs/src/components/BuiltinTools.mdx +++ b/docs/src/components/BuiltinTools.mdx @@ -10,6 +10,7 @@ import { LinkCard } from '@astrojs/starlight/components'; + diff --git a/docs/src/content/docs/reference/scripts/system.mdx b/docs/src/content/docs/reference/scripts/system.mdx index 5772557e86..f97f59743b 100644 --- a/docs/src/content/docs/reference/scripts/system.mdx +++ b/docs/src/content/docs/reference/scripts/system.mdx @@ -590,6 +590,73 @@ Emit type information compatible with PyLance.` ````` +### `system.python_interpreter` + +Python Dockerized code execution + + + +- tool `python_interpreter`: Executes python 3.12 code in a docker container. The process output is returned. Use 'print' to output data. + +`````js wrap title="system.python_interpreter" +system({ + title: "Python Dockerized code execution", +}) + +const image = env.vars.pythonImage ?? "python:3.12" + +let container = null + +defTool( + "python_interpreter", + "Executes python 3.12 code in a docker container. The process output is returned. Use 'print' to output data.", + { + type: "object", + properties: { + requirements: { + type: "string", + description: `list of pip packages to install using pip. should be using the pip install format: + + +`, + }, + main: { + type: "string", + description: "python 3.12 source code to execute", + }, + }, + required: ["requirements", "main"], + }, + async (args) => { + const { requirements, main = "" } = args + console.log(`python: running code...`) + container = await host.container({ image, networkEnabled: true }) + if (requirements) { + console.log(`installing: ` + requirements) + await container.writeText( + "requirements.txt", + requirements.replace(/[ ,]\s*/g, "\n") + ) + const res = await container.exec("pip", [ + "install", + "--root-user-action", + "ignore", + "-r", + "requirements.txt", + ]) + if (res.failed) throw new Error(`Failed to install requirements`) + } + + console.log(`code: ` + main) + await container.writeText("main.py", main) + const res = await container.exec("python", ["main.py"]) + return res + } +) + +````` + + ### `system.retrieval_fuzz_search` Full Text Fuzzy Search diff --git a/genaisrc/genaiscript.d.ts b/genaisrc/genaiscript.d.ts index 6e01d272c5..8fccf3c5fa 100644 --- a/genaisrc/genaiscript.d.ts +++ b/genaisrc/genaiscript.d.ts @@ -1,3 +1,5 @@ +type OptionsOrString = (string & {}) | TOptions + interface PromptGenerationConsole { log(...data: any[]): void warn(...data: any[]): void @@ -65,9 +67,9 @@ interface PromptLike extends PromptDefinition { text?: string } -type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" +type SystemPromptId = OptionsOrString<"system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot"> -type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" +type SystemToolId = OptionsOrString<"fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search"> type FileMergeHandler = ( filename: string, @@ -210,6 +212,7 @@ interface ScriptRuntimeOptions { * - `system.functions`: use functions * - `system.math`: Math expression evaluator * - `system.python`: Expert at generating and understanding Python code. +* - `system.python_interpreter`: Python Dockerized code execution * - `system.retrieval_fuzz_search`: Full Text Fuzzy Search * - `system.retrieval_vector_search`: Embeddings Vector Search * - `system.retrieval_web_search`: Web Search @@ -230,6 +233,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression +* - `python_interpreter`: Executes python 3.12 code in a docker container. The process output is returned. Use 'print' to output data. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. @@ -1622,6 +1626,11 @@ interface ContainerHost extends ShellHost { * @param toContainer directory in the container */ copyTo(fromHost: string | string[], toContainer: string): Promise + + /** + * Stops and cleans out the container + */ + stop(): Promise } interface PromptContext extends ChatGenerationContext { diff --git a/packages/cli/src/docker.ts b/packages/cli/src/docker.ts index ce19eaa1eb..af3885f5d3 100644 --- a/packages/cli/src/docker.ts +++ b/packages/cli/src/docker.ts @@ -13,8 +13,10 @@ import { errorMessage } from "../../core/src/error" import { host } from "../../core/src/host" import { installImport } from "../../core/src/import" import { TraceOptions } from "../../core/src/trace" -import { logError, dotGenaiscriptPath } from "../../core/src/util" +import { logError, dotGenaiscriptPath, logVerbose } from "../../core/src/util" import { CORE_VERSION } from "../../core/src/version" +import { YAMLStringify } from "../../core/src/yaml" +import { isQuiet } from "./log" type DockerodeType = import("dockerode") @@ -62,6 +64,30 @@ export class DockerManager { this.containers = [] } + async stopContainer(id: string) { + const c = await this._docker?.getContainer(id) + if (c) { + try { + await c.stop() + } catch {} + try { + await c.remove() + } catch (e) { + logError(e) + } + } + const i = this.containers.findIndex((c) => c.id === id) + if (i > -1) { + const container = this.containers[i] + try { + await remove(container.hostPath) + } catch (e) { + logError(e) + } + this.containers.splice(i, 1) + } + } + async checkImage(image: string) { await this.init() try { @@ -162,6 +188,10 @@ export class DockerManager { const inspection = await container.inspect() trace?.itemValue(`container state`, inspection.State?.Status) + const stop: () => Promise = async () => { + await this.stopContainer(container.id) + } + const exec: ShellHost["exec"] = async ( command, args, @@ -178,6 +208,10 @@ export class DockerManager { trace?.itemValue(`container`, container.id) trace?.itemValue(`cwd`, cwd) trace?.item(`\`${command}\` ${args.join(" ")}`) + if (!isQuiet) + logVerbose( + `container exec: ${command} ${args.join(" ")}` + ) let inspection = await container.inspect() trace?.itemValue( @@ -220,8 +254,14 @@ export class DockerManager { exitCode === 0, `exit code: ${sres.exitCode}` ) - if (sres.stdout) trace?.detailsFenced(`stdout`, sres.stdout) - if (sres.stderr) trace?.detailsFenced(`stderr`, sres.stderr) + if (sres.stdout) { + trace?.detailsFenced(`stdout`, sres.stdout, "txt") + if (!isQuiet) logVerbose(sres.stdout) + } + if (sres.stderr) { + trace?.detailsFenced(`stderr`, sres.stderr, "txt") + if (!isQuiet) logVerbose(sres.stderr) + } return sres } catch (e) { @@ -239,7 +279,9 @@ export class DockerManager { const writeText = async (filename: string, content: string) => { const hostFilename = host.path.resolve(hostPath, filename) await ensureDir(host.path.dirname(hostFilename)) - await writeFile(hostFilename, content, { encoding: "utf8" }) + await writeFile(hostFilename, content ?? "", { + encoding: "utf8", + }) } const readText = async (filename: string, content: string) => { @@ -262,6 +304,7 @@ export class DockerManager { disablePurge: !!options.disablePurge, hostPath, containerPath, + stop, exec, writeText, readText, diff --git a/packages/cli/src/run.ts b/packages/cli/src/run.ts index 66ff6b4eb7..6a98e15d4e 100644 --- a/packages/cli/src/run.ts +++ b/packages/cli/src/run.ts @@ -4,7 +4,6 @@ import { isQuiet } from "./log" import { emptyDir, ensureDir } from "fs-extra" import { convertDiagnosticsToSARIF } from "./sarif" import { buildProject } from "./build" -import { createProgressSpinner } from "./spinner" import { diagnosticsToCSV } from "../../core/src/ast" import { CancellationOptions } from "../../core/src/cancellation" import { ChatCompletionsProgressReport } from "../../core/src/chattypes" @@ -109,13 +108,8 @@ export async function runScript( const cancellationToken = options.cancellationToken const jsSource = options.jsSource - const spinner = - !stream && !isQuiet - ? createProgressSpinner(`preparing tools in ${process.cwd()}`) - : undefined const fail = (msg: string, exitCode: number) => { - if (spinner) spinner.fail(msg) - else logVerbose(msg) + logVerbose(msg) return { exitCode, result } } @@ -130,7 +124,7 @@ export async function runScript( const ffs = await host.findFiles(arg, { applyGitIgnore: excludeGitIgnore, }) - if (!ffs.length) { + if (!ffs?.length) { return fail( `no files matching ${arg}`, FILES_NOT_FOUND_ERROR_CODE @@ -188,8 +182,7 @@ export async function runScript( infoCb: (args) => { const { text } = args if (text) { - if (spinner) spinner.start(text) - else if (!isQuiet) logVerbose(text) + if (!isQuiet) logVerbose(text) infoCb?.(args) } }, @@ -197,7 +190,7 @@ export async function runScript( const { responseChunk, tokensSoFar } = args tokens = tokensSoFar if (stream && responseChunk) process.stdout.write(responseChunk) - if (spinner) spinner.report({ count: tokens }) + else if (!isQuiet) process.stderr.write(responseChunk) partialCb?.(args) }, skipLLM, @@ -230,18 +223,13 @@ export async function runScript( }, }) } catch (err) { - if (spinner) spinner.fail() if (isCancelError(err)) return fail("user cancelled", USER_CANCELLED_ERROR_CODE) logError(err) return fail("runtime error", RUNTIME_ERROR_CODE) } if (!isQuiet) logVerbose("") // force new line - if (spinner) { - if (result.status !== "success") - spinner.fail(`${spinner.text}, ${result.statusText}`) - else spinner.succeed() - } else if (result.status !== "success") + if (result.status !== "success") logVerbose(result.statusText ?? result.status) if (outTrace) await writeText(outTrace, trace.content) @@ -434,7 +422,6 @@ export async function runScript( if (failOnErrors && result.annotations?.some((a) => a.severity === "error")) return fail("error annotations found", ANNOTATION_ERROR_CODE) - spinner?.stop() process.stderr.write("\n") return { exitCode: 0, result } } diff --git a/packages/core/src/chat.ts b/packages/core/src/chat.ts index 54fabbf167..4f41af8334 100644 --- a/packages/core/src/chat.ts +++ b/packages/core/src/chat.ts @@ -2,13 +2,13 @@ import { MarkdownTrace } from "./trace" import { PromptImage, renderPromptNode } from "./promptdom" import { LanguageModelConfiguration, host } from "./host" import { GenerationOptions } from "./generation" -import { JSON5TryParse, JSON5parse, isJSONObjectOrArray } from "./json5" +import { JSON5parse, JSONLLMTryParse, isJSONObjectOrArray } from "./json5" import { CancellationOptions, CancellationToken, checkCancelled, } from "./cancellation" -import { assert } from "./util" +import { assert, logError } from "./util" import { extractFenced, findFirstDataFence } from "./fence" import { toStrictJSONSchema, @@ -18,7 +18,6 @@ import { import { MAX_DATA_REPAIRS, MAX_TOOL_CALLS } from "./constants" import { parseAnnotations } from "./annotations" import { errorMessage, isCancelError, serializeError } from "./error" -import { YAMLStringify } from "./yaml" import { estimateChatTokens } from "./chatencoder" import { createChatTurnGenerationContext } from "./runpromptcontext" import { dedent } from "./indent" @@ -141,10 +140,11 @@ async function runToolCalls( checkCancelled(cancellationToken) trace.startDetails(`📠 tool call ${call.name}`) try { - const callArgs: any = call.arguments - ? JSON5TryParse(call.arguments) + const callArgs: any = call.arguments // sometimes wrapped in \`\`\`json ... + ? JSONLLMTryParse(call.arguments) : undefined - trace.itemValue(`args`, callArgs ?? call.arguments) + trace.fence(call.arguments, "json") + if (callArgs === undefined) trace.error("arguments failed to parse") const fd = functions.find((f) => f.definition.name === call.name) if (!fd) throw new Error(`tool ${call.name} not found`) @@ -207,6 +207,7 @@ ${fenceMD(content, " ")} tool_call_id: call.id, }) } catch (e) { + logError(e) trace.error(`tool call ${call.id} error`, e) throw e } finally { @@ -238,7 +239,7 @@ async function applyRepairs( const invalids = fences.filter((f) => f?.validation?.valid === false) if (responseSchema) { - const value = JSON5TryParse(content) + const value = JSONLLMTryParse(content) const schema = promptParametersSchemaToJSONSchema(responseSchema) const res = validateJSONWithSchema(value, schema, { trace }) if (!res.valid) @@ -347,7 +348,7 @@ function structurifyChatSession( } } else { json = isJSONObjectOrArray(text) - ? JSON5TryParse(text, undefined) + ? JSONLLMTryParse(text) : (undefined ?? findFirstDataFence(fences)) } const frames: DataFrame[] = [] @@ -512,10 +513,11 @@ export async function executeChatSession( infoCb?.({ text: `prompting ${model} (~${estimateChatTokens(model, messages)} tokens)`, }) - trace.details( - `💬 messages (${messages.length})`, - renderMessagesToMarkdown(messages) - ) + if (messages) + trace.details( + `💬 messages (${messages.length})`, + renderMessagesToMarkdown(messages) + ) // make request let resp: ChatCompletionResponse diff --git a/packages/core/src/csv.test.ts b/packages/core/src/csv.test.ts index 1176b6669c..44df058a21 100644 --- a/packages/core/src/csv.test.ts +++ b/packages/core/src/csv.test.ts @@ -1,16 +1,74 @@ -import { CSVParse } from './csv' -import { describe, test } from 'node:test' -import assert from 'node:assert/strict' +import { describe, test, beforeEach } from "node:test"; +import assert from "node:assert/strict"; +import { CSVParse, CSVTryParse, CSVToMarkdown } from "./csv"; -describe('csv', () => { - test('parses simple CSV with default options', () => { - const csvText = 'name,age\nAlice,30\nBob,25' - const result = CSVParse(csvText) - assert.deepStrictEqual(result, [ - { name: 'Alice', age: '30' }, - { name: 'Bob', age: '25' } - ]) - }) +describe('CSVParse', () => { + test('Parse simple CSV data with default options', () => { + const csv = "name,age\nJohn,30\nJane,25"; + const result = CSVParse(csv); + assert.deepEqual(result, [ + { name: "John", age: "30" }, + { name: "Jane", age: "25" } + ]); + }); - // Additional tests for CSVParse... -}) + test('Parse CSV data with custom delimiter', () => { + const csv = "name|age\nJohn|30\nJane|25"; + const result = CSVParse(csv, { delimiter: "|" }); + assert.deepEqual(result, [ + { name: "John", age: "30" }, + { name: "Jane", age: "25" } + ]); + }); + + test('Parse CSV data with specified headers', () => { + const csv = "John,30\nJane,25"; + const result = CSVParse(csv, { headers: ["name", "age"] }); + assert.deepEqual(result, [ + { name: "John", age: "30" }, + { name: "Jane", age: "25" } + ]); + }); +}); + +describe('CSVTryParse', () => { + test('Try to parse valid CSV data', () => { + const csv = "name,age\nJohn,30\nJane,25"; + const result = CSVTryParse(csv); + assert.deepEqual(result, [ + { name: "John", age: "30" }, + { name: "Jane", age: "25" } + ]); + }); +}); + +describe('CSVToMarkdown', () => { + test('Convert parsed CSV data to markdown table', () => { + const csv = [{ name: "John", age: "30" }, { name: "Jane", age: "25" }]; + const result = CSVToMarkdown(csv); + const expected = ` +|name|age| +|-|-| +|John|30| +|Jane|25| +`.trim().replace(/[\t ]+/g, " "); + assert.equal(result, expected); + }); + + test('Convert parsed CSV data to markdown table with custom headers', () => { + const csv = [{ name: "John", age: "30" }, { name: "Jane", age: "25" }]; + const result = CSVToMarkdown(csv, { headers: ["age", "name"] }); + const expected = ` +|age|name| +|-|-| +|30|John| +|25|Jane| +`.trim().replace(/[\t ]+/g, " "); + assert.equal(result, expected); + }); + + test('Handle empty CSV data input', () => { + const result = CSVToMarkdown([]); + assert.equal(result, ""); + }); +}); \ No newline at end of file diff --git a/packages/core/src/csv.ts b/packages/core/src/csv.ts index 75d8093a46..87923ccec8 100644 --- a/packages/core/src/csv.ts +++ b/packages/core/src/csv.ts @@ -27,7 +27,7 @@ export function CSVTryParse( delimiter?: string headers?: string[] } & TraceOptions -): object[] { +): object[] | undefined { const { trace } = options || {} try { return CSVParse(text, options) @@ -51,8 +51,9 @@ export function CSVToMarkdown(csv: object[], options?: { headers?: string[] }) { ), ] const md = markdownTable(table, { - align: "left", stringLength: (str) => str.length, + padding: false, + alignDelimiters: false, }) // improves LLM performance const mdcompact = md.replace(/[\t ]+/g, " ") diff --git a/packages/core/src/expander.ts b/packages/core/src/expander.ts index 92a84fcf2f..1e8438cf20 100644 --- a/packages/core/src/expander.ts +++ b/packages/core/src/expander.ts @@ -164,6 +164,7 @@ export async function expandTemplate( ) { const model = options.model assert(!!model) + const messages: ChatCompletionMessageParam[] = [] const cancellationToken = options.cancellationToken const systems = resolveSystems(prj, template) const systemTemplates = systems.map((s) => prj.getTemplate(s)) @@ -189,7 +190,7 @@ export async function expandTemplate( normalizeInt(env.vars["maxToolCalls"]) ?? normalizeInt(env.vars["max_tool_calls"]) ?? template.maxToolCalls ?? - MAX_TOOL_CALLS + MAX_TOOL_CALLS let seed = options.seed ?? normalizeInt(env.vars["seed"]) ?? template.seed if (seed !== undefined) seed = seed >> 0 @@ -222,23 +223,22 @@ export async function expandTemplate( if (prompt.status !== "success" || prompt.text === "") // cancelled - return { status: prompt.status, statusText: prompt.statusText } + return { status: prompt.status, statusText: prompt.statusText, messages } if (cancellationToken?.isCancellationRequested) - return { status: "cancelled", statusText: "user cancelled" } + return { status: "cancelled", statusText: "user cancelled", messages } const systemMessage: ChatCompletionSystemMessageParam = { role: "system", content: "", } - const messages: ChatCompletionMessageParam[] = [] if (prompt.text) messages.push(toChatCompletionUserMessage(prompt.text, prompt.images)) if (prompt.aici) messages.push(prompt.aici) for (let i = 0; i < systems.length; ++i) { if (cancellationToken?.isCancellationRequested) - return { status: "cancelled", statusText: "user cancelled" } + return { status: "cancelled", statusText: "user cancelled", messages } let systemTemplate = systems[i] let system = prj.getTemplate(systemTemplate) @@ -277,7 +277,7 @@ export async function expandTemplate( trace.endDetails() if (sysr.status !== "success") - return { status: sysr.status, statusText: sysr.statusText } + return { status: sysr.status, statusText: sysr.statusText, messages } } const responseSchema = promptParametersSchemaToJSONSchema( diff --git a/packages/core/src/fetch.ts b/packages/core/src/fetch.ts index bcb20f1058..b46a5aa5fb 100644 --- a/packages/core/src/fetch.ts +++ b/packages/core/src/fetch.ts @@ -74,7 +74,7 @@ export function traceFetchPost( ${Object.entries(headers) .map(([k, v]) => `-H "${k}: ${v}" \\`) .join("\n")} --d '${JSON.stringify(body).replace(/'/g, "'\\''")}' +-d '${JSON.stringify(body).replace(/'/g, "'\\''").replace(/\r\n/g, "\n \\")}' ` if (trace) trace.detailsFenced(`✉️ fetch`, cmd, "bash") else logVerbose(cmd) diff --git a/packages/core/src/genaisrc/genaiscript.d.ts b/packages/core/src/genaisrc/genaiscript.d.ts index 6e01d272c5..8fccf3c5fa 100644 --- a/packages/core/src/genaisrc/genaiscript.d.ts +++ b/packages/core/src/genaisrc/genaiscript.d.ts @@ -1,3 +1,5 @@ +type OptionsOrString = (string & {}) | TOptions + interface PromptGenerationConsole { log(...data: any[]): void warn(...data: any[]): void @@ -65,9 +67,9 @@ interface PromptLike extends PromptDefinition { text?: string } -type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" +type SystemPromptId = OptionsOrString<"system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot"> -type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" +type SystemToolId = OptionsOrString<"fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search"> type FileMergeHandler = ( filename: string, @@ -210,6 +212,7 @@ interface ScriptRuntimeOptions { * - `system.functions`: use functions * - `system.math`: Math expression evaluator * - `system.python`: Expert at generating and understanding Python code. +* - `system.python_interpreter`: Python Dockerized code execution * - `system.retrieval_fuzz_search`: Full Text Fuzzy Search * - `system.retrieval_vector_search`: Embeddings Vector Search * - `system.retrieval_web_search`: Web Search @@ -230,6 +233,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression +* - `python_interpreter`: Executes python 3.12 code in a docker container. The process output is returned. Use 'print' to output data. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. @@ -1622,6 +1626,11 @@ interface ContainerHost extends ShellHost { * @param toContainer directory in the container */ copyTo(fromHost: string | string[], toContainer: string): Promise + + /** + * Stops and cleans out the container + */ + stop(): Promise } interface PromptContext extends ChatGenerationContext { diff --git a/packages/core/src/genaisrc/system.python_interpreter.genai.js b/packages/core/src/genaisrc/system.python_interpreter.genai.js new file mode 100644 index 0000000000..8e886116b2 --- /dev/null +++ b/packages/core/src/genaisrc/system.python_interpreter.genai.js @@ -0,0 +1,54 @@ +system({ + title: "Python Dockerized code execution", +}) + +const image = env.vars.pythonImage ?? "python:3.12" + +let container = null + +defTool( + "python_interpreter", + "Executes python 3.12 code in a docker container. The process output is returned. Use 'print' to output data.", + { + type: "object", + properties: { + requirements: { + type: "string", + description: `list of pip packages to install using pip. should be using the pip install format: + + +`, + }, + main: { + type: "string", + description: "python 3.12 source code to execute", + }, + }, + required: ["requirements", "main"], + }, + async (args) => { + const { requirements, main = "" } = args + console.log(`python: running code...`) + container = await host.container({ image, networkEnabled: true }) + if (requirements) { + console.log(`installing: ` + requirements) + await container.writeText( + "requirements.txt", + requirements.replace(/[ ,]\s*/g, "\n") + ) + const res = await container.exec("pip", [ + "install", + "--root-user-action", + "ignore", + "-r", + "requirements.txt", + ]) + if (res.failed) throw new Error(`Failed to install requirements`) + } + + console.log(`code: ` + main) + await container.writeText("main.py", main) + const res = await container.exec("python", ["main.py"]) + return res + } +) diff --git a/packages/core/src/json5.ts b/packages/core/src/json5.ts index d28d877455..5d60d5aadc 100644 --- a/packages/core/src/json5.ts +++ b/packages/core/src/json5.ts @@ -52,3 +52,13 @@ export function JSON5TryParse( repair: true, }) } + +const startRx = /^\s*\`\`\`json\s/ +const endRx = /\`\`\`\s*$/ +export function JSONLLMTryParse(s: string): any { + if (s === undefined || s === null) return s + if (startRx.test(s) && endRx.test(s)) + s = s.replace(startRx, "").replace(endRx, "") + return JSON5TryParse(s) +} + diff --git a/packages/core/src/liner.test.ts b/packages/core/src/liner.test.ts index 47154f43d1..5d267ecd51 100644 --- a/packages/core/src/liner.test.ts +++ b/packages/core/src/liner.test.ts @@ -47,8 +47,8 @@ index 8cf2f17f..e17283d9 100644 [4] +line 3 ` assertDiff(diff, expected) - }) - + }) + test("diff test 3", function () { const diff = `diff --git a/packages/core/src/liner.diff.txt b/packages/core/src/liner.diff.txt index 8cf2f17f..519f67a6 100644 @@ -71,7 +71,7 @@ index 8cf2f17f..519f67a6 100644 [4] line 3 ` assertDiff(diff, expected) - }) + }) test("diff test 4", function () { const diff = `diff --git a/packages/core/src/liner.ts b/packages/core/src/liner.ts @@ -100,7 +100,7 @@ index 1215f7e7..385884e0 100644 [37] } ` assertDiff(diff, expected) - }) + }) test("returns the original diff if it is empty", function () { const diff = "" const result = llmifyDiff(diff) @@ -109,10 +109,14 @@ index 1215f7e7..385884e0 100644 }) function assertDiff(diff: string, expected: string) { const result = llmifyDiff(diff) - console.log(diff) - console.log("\n> result") - console.log(result) - console.log("\n> expected") - console.log(expected) - assert.strictEqual(result, expected) + try { + assert.strictEqual(result, expected) + } catch (e) { + console.log(diff) + console.log("\n> result") + console.log(result) + console.log("\n> expected") + console.log(expected) + throw e + } } diff --git a/packages/core/src/promptrunner.ts b/packages/core/src/promptrunner.ts index 67208fef9b..6293e4a1d4 100644 --- a/packages/core/src/promptrunner.ts +++ b/packages/core/src/promptrunner.ts @@ -119,7 +119,7 @@ export async function runTemplate( ) // if the expansion failed, show the user the trace - if (status !== "success") { + if (status !== "success" || !messages.length) { trace.renderErrors() return { status, diff --git a/packages/core/src/scripts.ts b/packages/core/src/scripts.ts index 9bbb11248d..f6e749627c 100644 --- a/packages/core/src/scripts.ts +++ b/packages/core/src/scripts.ts @@ -49,11 +49,11 @@ export async function fixPromptDefinitions(project: Project) { // update the system prompt identifiers defContent = defContent .replace( - "type SystemPromptId = string", - `type SystemPromptId = ${systems + "type SystemPromptId = OptionsOrString", + `type SystemPromptId = OptionsOrString<${systems .sort((a, b) => a.id.localeCompare(b.id)) .map((s) => JSON.stringify(s.id)) - .join(" | ")}` + .join(" | ")}>` ) .replace( " system?: SystemPromptId[]", @@ -67,11 +67,11 @@ ${systems.map((s) => `* - \`${s.id}\`: ${s.title || s.description}`).join("\n")} // update the tool prompt identifiers defContent = defContent .replace( - "type SystemToolId = string", - `type SystemToolId = ${tools + "type SystemToolId = OptionsOrString", + `type SystemToolId = OptionsOrString<${tools .sort((a, b) => a.name.localeCompare(b.name)) .map((s) => JSON.stringify(s.name)) - .join(" | ")}` + .join(" | ")}>` ) .replace( " tools?: SystemToolId[]", diff --git a/packages/core/src/types/prompt_template.d.ts b/packages/core/src/types/prompt_template.d.ts index 8865999995..92973d4113 100644 --- a/packages/core/src/types/prompt_template.d.ts +++ b/packages/core/src/types/prompt_template.d.ts @@ -1,3 +1,5 @@ +type OptionsOrString = (string & {}) | TOptions + interface PromptGenerationConsole { log(...data: any[]): void warn(...data: any[]): void @@ -65,9 +67,9 @@ interface PromptLike extends PromptDefinition { text?: string } -type SystemPromptId = string +type SystemPromptId = OptionsOrString -type SystemToolId = string +type SystemToolId = OptionsOrString type FileMergeHandler = ( filename: string, @@ -1587,6 +1589,11 @@ interface ContainerHost extends ShellHost { * @param toContainer directory in the container */ copyTo(fromHost: string | string[], toContainer: string): Promise + + /** + * Stops and cleans out the container + */ + stop(): Promise } interface PromptContext extends ChatGenerationContext { diff --git a/packages/core/src/util.ts b/packages/core/src/util.ts index 43ec7fc7db..ef81bf25dd 100644 --- a/packages/core/src/util.ts +++ b/packages/core/src/util.ts @@ -180,10 +180,10 @@ export function logWarn(msg: string) { export function logError(msg: string | Error | SerializedError) { const { message, ...e } = serializeError(msg) if (message) host.log(LogLevel.Error, message) + console.debug(msg) const se = YAMLStringify(e) if (!/^\s*\{\}\s*$/) host.log(LogLevel.Info, se) } - export function concatArrays(...arrays: T[][]): T[] { if (arrays.length == 0) return [] return arrays[0].concat(...arrays.slice(1)) diff --git a/packages/sample/genaisrc/data-analyst.genai.mjs b/packages/sample/genaisrc/data-analyst.genai.mjs new file mode 100644 index 0000000000..93a9b24ccd --- /dev/null +++ b/packages/sample/genaisrc/data-analyst.genai.mjs @@ -0,0 +1,12 @@ +script({ + system: ["system", "system.python_interpreter"], + files: ["src/penguins.csv"], +}) + +const data = def("DATA", env.files, { sliceSample: 25 }) + +$`Analyze ${data} with a detailed statistical analysis. + +- Do not generate visualizations. +- Validate computations with code. +` diff --git a/packages/sample/genaisrc/genaiscript.d.ts b/packages/sample/genaisrc/genaiscript.d.ts index 6e01d272c5..8fccf3c5fa 100644 --- a/packages/sample/genaisrc/genaiscript.d.ts +++ b/packages/sample/genaisrc/genaiscript.d.ts @@ -1,3 +1,5 @@ +type OptionsOrString = (string & {}) | TOptions + interface PromptGenerationConsole { log(...data: any[]): void warn(...data: any[]): void @@ -65,9 +67,9 @@ interface PromptLike extends PromptDefinition { text?: string } -type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" +type SystemPromptId = OptionsOrString<"system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot"> -type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" +type SystemToolId = OptionsOrString<"fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search"> type FileMergeHandler = ( filename: string, @@ -210,6 +212,7 @@ interface ScriptRuntimeOptions { * - `system.functions`: use functions * - `system.math`: Math expression evaluator * - `system.python`: Expert at generating and understanding Python code. +* - `system.python_interpreter`: Python Dockerized code execution * - `system.retrieval_fuzz_search`: Full Text Fuzzy Search * - `system.retrieval_vector_search`: Embeddings Vector Search * - `system.retrieval_web_search`: Web Search @@ -230,6 +233,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression +* - `python_interpreter`: Executes python 3.12 code in a docker container. The process output is returned. Use 'print' to output data. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. @@ -1622,6 +1626,11 @@ interface ContainerHost extends ShellHost { * @param toContainer directory in the container */ copyTo(fromHost: string | string[], toContainer: string): Promise + + /** + * Stops and cleans out the container + */ + stop(): Promise } interface PromptContext extends ChatGenerationContext { diff --git a/packages/sample/genaisrc/node/genaiscript.d.ts b/packages/sample/genaisrc/node/genaiscript.d.ts index 6e01d272c5..8fccf3c5fa 100644 --- a/packages/sample/genaisrc/node/genaiscript.d.ts +++ b/packages/sample/genaisrc/node/genaiscript.d.ts @@ -1,3 +1,5 @@ +type OptionsOrString = (string & {}) | TOptions + interface PromptGenerationConsole { log(...data: any[]): void warn(...data: any[]): void @@ -65,9 +67,9 @@ interface PromptLike extends PromptDefinition { text?: string } -type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" +type SystemPromptId = OptionsOrString<"system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot"> -type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" +type SystemToolId = OptionsOrString<"fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search"> type FileMergeHandler = ( filename: string, @@ -210,6 +212,7 @@ interface ScriptRuntimeOptions { * - `system.functions`: use functions * - `system.math`: Math expression evaluator * - `system.python`: Expert at generating and understanding Python code. +* - `system.python_interpreter`: Python Dockerized code execution * - `system.retrieval_fuzz_search`: Full Text Fuzzy Search * - `system.retrieval_vector_search`: Embeddings Vector Search * - `system.retrieval_web_search`: Web Search @@ -230,6 +233,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression +* - `python_interpreter`: Executes python 3.12 code in a docker container. The process output is returned. Use 'print' to output data. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. @@ -1622,6 +1626,11 @@ interface ContainerHost extends ShellHost { * @param toContainer directory in the container */ copyTo(fromHost: string | string[], toContainer: string): Promise + + /** + * Stops and cleans out the container + */ + stop(): Promise } interface PromptContext extends ChatGenerationContext { diff --git a/packages/sample/genaisrc/pandas.genai.mjs b/packages/sample/genaisrc/pandas.genai.mjs new file mode 100644 index 0000000000..64d5ba6ee5 --- /dev/null +++ b/packages/sample/genaisrc/pandas.genai.mjs @@ -0,0 +1,19 @@ +script({ tests: {} }) +const container = await host.container({ + image: "python:3.12", + networkEnabled: true, +}) +await container.exec("pip", [ + "install", + "--root-user-action", + "ignore", + "pandas", +]) +await container.writeText( + "main.py", + `import pandas as pd +print(pd) +` +) +const res = await container.exec("python", ["main.py"]) +if (res.failed) throw new Error("import failed") diff --git a/packages/sample/genaisrc/python/genaiscript.d.ts b/packages/sample/genaisrc/python/genaiscript.d.ts index 6e01d272c5..8fccf3c5fa 100644 --- a/packages/sample/genaisrc/python/genaiscript.d.ts +++ b/packages/sample/genaisrc/python/genaiscript.d.ts @@ -1,3 +1,5 @@ +type OptionsOrString = (string & {}) | TOptions + interface PromptGenerationConsole { log(...data: any[]): void warn(...data: any[]): void @@ -65,9 +67,9 @@ interface PromptLike extends PromptDefinition { text?: string } -type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" +type SystemPromptId = OptionsOrString<"system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot"> -type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" +type SystemToolId = OptionsOrString<"fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search"> type FileMergeHandler = ( filename: string, @@ -210,6 +212,7 @@ interface ScriptRuntimeOptions { * - `system.functions`: use functions * - `system.math`: Math expression evaluator * - `system.python`: Expert at generating and understanding Python code. +* - `system.python_interpreter`: Python Dockerized code execution * - `system.retrieval_fuzz_search`: Full Text Fuzzy Search * - `system.retrieval_vector_search`: Embeddings Vector Search * - `system.retrieval_web_search`: Web Search @@ -230,6 +233,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression +* - `python_interpreter`: Executes python 3.12 code in a docker container. The process output is returned. Use 'print' to output data. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. @@ -1622,6 +1626,11 @@ interface ContainerHost extends ShellHost { * @param toContainer directory in the container */ copyTo(fromHost: string | string[], toContainer: string): Promise + + /** + * Stops and cleans out the container + */ + stop(): Promise } interface PromptContext extends ChatGenerationContext { diff --git a/packages/sample/genaisrc/style/genaiscript.d.ts b/packages/sample/genaisrc/style/genaiscript.d.ts index 6e01d272c5..8fccf3c5fa 100644 --- a/packages/sample/genaisrc/style/genaiscript.d.ts +++ b/packages/sample/genaisrc/style/genaiscript.d.ts @@ -1,3 +1,5 @@ +type OptionsOrString = (string & {}) | TOptions + interface PromptGenerationConsole { log(...data: any[]): void warn(...data: any[]): void @@ -65,9 +67,9 @@ interface PromptLike extends PromptDefinition { text?: string } -type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" +type SystemPromptId = OptionsOrString<"system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot"> -type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" +type SystemToolId = OptionsOrString<"fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search"> type FileMergeHandler = ( filename: string, @@ -210,6 +212,7 @@ interface ScriptRuntimeOptions { * - `system.functions`: use functions * - `system.math`: Math expression evaluator * - `system.python`: Expert at generating and understanding Python code. +* - `system.python_interpreter`: Python Dockerized code execution * - `system.retrieval_fuzz_search`: Full Text Fuzzy Search * - `system.retrieval_vector_search`: Embeddings Vector Search * - `system.retrieval_web_search`: Web Search @@ -230,6 +233,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression +* - `python_interpreter`: Executes python 3.12 code in a docker container. The process output is returned. Use 'print' to output data. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. @@ -1622,6 +1626,11 @@ interface ContainerHost extends ShellHost { * @param toContainer directory in the container */ copyTo(fromHost: string | string[], toContainer: string): Promise + + /** + * Stops and cleans out the container + */ + stop(): Promise } interface PromptContext extends ChatGenerationContext { diff --git a/packages/sample/src/aici/genaiscript.d.ts b/packages/sample/src/aici/genaiscript.d.ts index 6e01d272c5..8fccf3c5fa 100644 --- a/packages/sample/src/aici/genaiscript.d.ts +++ b/packages/sample/src/aici/genaiscript.d.ts @@ -1,3 +1,5 @@ +type OptionsOrString = (string & {}) | TOptions + interface PromptGenerationConsole { log(...data: any[]): void warn(...data: any[]): void @@ -65,9 +67,9 @@ interface PromptLike extends PromptDefinition { text?: string } -type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" +type SystemPromptId = OptionsOrString<"system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot"> -type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" +type SystemToolId = OptionsOrString<"fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search"> type FileMergeHandler = ( filename: string, @@ -210,6 +212,7 @@ interface ScriptRuntimeOptions { * - `system.functions`: use functions * - `system.math`: Math expression evaluator * - `system.python`: Expert at generating and understanding Python code. +* - `system.python_interpreter`: Python Dockerized code execution * - `system.retrieval_fuzz_search`: Full Text Fuzzy Search * - `system.retrieval_vector_search`: Embeddings Vector Search * - `system.retrieval_web_search`: Web Search @@ -230,6 +233,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression +* - `python_interpreter`: Executes python 3.12 code in a docker container. The process output is returned. Use 'print' to output data. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. @@ -1622,6 +1626,11 @@ interface ContainerHost extends ShellHost { * @param toContainer directory in the container */ copyTo(fromHost: string | string[], toContainer: string): Promise + + /** + * Stops and cleans out the container + */ + stop(): Promise } interface PromptContext extends ChatGenerationContext { diff --git a/packages/sample/src/errors/genaiscript.d.ts b/packages/sample/src/errors/genaiscript.d.ts index 6e01d272c5..8fccf3c5fa 100644 --- a/packages/sample/src/errors/genaiscript.d.ts +++ b/packages/sample/src/errors/genaiscript.d.ts @@ -1,3 +1,5 @@ +type OptionsOrString = (string & {}) | TOptions + interface PromptGenerationConsole { log(...data: any[]): void warn(...data: any[]): void @@ -65,9 +67,9 @@ interface PromptLike extends PromptDefinition { text?: string } -type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" +type SystemPromptId = OptionsOrString<"system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot"> -type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" +type SystemToolId = OptionsOrString<"fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search"> type FileMergeHandler = ( filename: string, @@ -210,6 +212,7 @@ interface ScriptRuntimeOptions { * - `system.functions`: use functions * - `system.math`: Math expression evaluator * - `system.python`: Expert at generating and understanding Python code. +* - `system.python_interpreter`: Python Dockerized code execution * - `system.retrieval_fuzz_search`: Full Text Fuzzy Search * - `system.retrieval_vector_search`: Embeddings Vector Search * - `system.retrieval_web_search`: Web Search @@ -230,6 +233,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression +* - `python_interpreter`: Executes python 3.12 code in a docker container. The process output is returned. Use 'print' to output data. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. @@ -1622,6 +1626,11 @@ interface ContainerHost extends ShellHost { * @param toContainer directory in the container */ copyTo(fromHost: string | string[], toContainer: string): Promise + + /** + * Stops and cleans out the container + */ + stop(): Promise } interface PromptContext extends ChatGenerationContext { diff --git a/packages/sample/src/makecode/genaiscript.d.ts b/packages/sample/src/makecode/genaiscript.d.ts index 6e01d272c5..8fccf3c5fa 100644 --- a/packages/sample/src/makecode/genaiscript.d.ts +++ b/packages/sample/src/makecode/genaiscript.d.ts @@ -1,3 +1,5 @@ +type OptionsOrString = (string & {}) | TOptions + interface PromptGenerationConsole { log(...data: any[]): void warn(...data: any[]): void @@ -65,9 +67,9 @@ interface PromptLike extends PromptDefinition { text?: string } -type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" +type SystemPromptId = OptionsOrString<"system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot"> -type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" +type SystemToolId = OptionsOrString<"fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search"> type FileMergeHandler = ( filename: string, @@ -210,6 +212,7 @@ interface ScriptRuntimeOptions { * - `system.functions`: use functions * - `system.math`: Math expression evaluator * - `system.python`: Expert at generating and understanding Python code. +* - `system.python_interpreter`: Python Dockerized code execution * - `system.retrieval_fuzz_search`: Full Text Fuzzy Search * - `system.retrieval_vector_search`: Embeddings Vector Search * - `system.retrieval_web_search`: Web Search @@ -230,6 +233,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression +* - `python_interpreter`: Executes python 3.12 code in a docker container. The process output is returned. Use 'print' to output data. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. @@ -1622,6 +1626,11 @@ interface ContainerHost extends ShellHost { * @param toContainer directory in the container */ copyTo(fromHost: string | string[], toContainer: string): Promise + + /** + * Stops and cleans out the container + */ + stop(): Promise } interface PromptContext extends ChatGenerationContext { diff --git a/packages/sample/src/tla/genaiscript.d.ts b/packages/sample/src/tla/genaiscript.d.ts index 6e01d272c5..8fccf3c5fa 100644 --- a/packages/sample/src/tla/genaiscript.d.ts +++ b/packages/sample/src/tla/genaiscript.d.ts @@ -1,3 +1,5 @@ +type OptionsOrString = (string & {}) | TOptions + interface PromptGenerationConsole { log(...data: any[]): void warn(...data: any[]): void @@ -65,9 +67,9 @@ interface PromptLike extends PromptDefinition { text?: string } -type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" +type SystemPromptId = OptionsOrString<"system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot"> -type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" +type SystemToolId = OptionsOrString<"fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search"> type FileMergeHandler = ( filename: string, @@ -210,6 +212,7 @@ interface ScriptRuntimeOptions { * - `system.functions`: use functions * - `system.math`: Math expression evaluator * - `system.python`: Expert at generating and understanding Python code. +* - `system.python_interpreter`: Python Dockerized code execution * - `system.retrieval_fuzz_search`: Full Text Fuzzy Search * - `system.retrieval_vector_search`: Embeddings Vector Search * - `system.retrieval_web_search`: Web Search @@ -230,6 +233,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression +* - `python_interpreter`: Executes python 3.12 code in a docker container. The process output is returned. Use 'print' to output data. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. @@ -1622,6 +1626,11 @@ interface ContainerHost extends ShellHost { * @param toContainer directory in the container */ copyTo(fromHost: string | string[], toContainer: string): Promise + + /** + * Stops and cleans out the container + */ + stop(): Promise } interface PromptContext extends ChatGenerationContext { diff --git a/packages/sample/src/vision/genaiscript.d.ts b/packages/sample/src/vision/genaiscript.d.ts index 6e01d272c5..8fccf3c5fa 100644 --- a/packages/sample/src/vision/genaiscript.d.ts +++ b/packages/sample/src/vision/genaiscript.d.ts @@ -1,3 +1,5 @@ +type OptionsOrString = (string & {}) | TOptions + interface PromptGenerationConsole { log(...data: any[]): void warn(...data: any[]): void @@ -65,9 +67,9 @@ interface PromptLike extends PromptDefinition { text?: string } -type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" +type SystemPromptId = OptionsOrString<"system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot"> -type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" +type SystemToolId = OptionsOrString<"fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search"> type FileMergeHandler = ( filename: string, @@ -210,6 +212,7 @@ interface ScriptRuntimeOptions { * - `system.functions`: use functions * - `system.math`: Math expression evaluator * - `system.python`: Expert at generating and understanding Python code. +* - `system.python_interpreter`: Python Dockerized code execution * - `system.retrieval_fuzz_search`: Full Text Fuzzy Search * - `system.retrieval_vector_search`: Embeddings Vector Search * - `system.retrieval_web_search`: Web Search @@ -230,6 +233,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression +* - `python_interpreter`: Executes python 3.12 code in a docker container. The process output is returned. Use 'print' to output data. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. @@ -1622,6 +1626,11 @@ interface ContainerHost extends ShellHost { * @param toContainer directory in the container */ copyTo(fromHost: string | string[], toContainer: string): Promise + + /** + * Stops and cleans out the container + */ + stop(): Promise } interface PromptContext extends ChatGenerationContext { diff --git a/slides/genaisrc/genaiscript.d.ts b/slides/genaisrc/genaiscript.d.ts index 6e01d272c5..8fccf3c5fa 100644 --- a/slides/genaisrc/genaiscript.d.ts +++ b/slides/genaisrc/genaiscript.d.ts @@ -1,3 +1,5 @@ +type OptionsOrString = (string & {}) | TOptions + interface PromptGenerationConsole { log(...data: any[]): void warn(...data: any[]): void @@ -65,9 +67,9 @@ interface PromptLike extends PromptDefinition { text?: string } -type SystemPromptId = "system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot" +type SystemPromptId = OptionsOrString<"system" | "system.annotations" | "system.changelog" | "system.diagrams" | "system.diff" | "system.explanations" | "system.files" | "system.files_schema" | "system.fs_find_files" | "system.fs_read_file" | "system.fs_read_summary" | "system.functions" | "system.math" | "system.python" | "system.python_interpreter" | "system.retrieval_fuzz_search" | "system.retrieval_vector_search" | "system.retrieval_web_search" | "system.schema" | "system.tasks" | "system.technical" | "system.typescript" | "system.zero_shot_cot"> -type SystemToolId = "fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search" +type SystemToolId = OptionsOrString<"fs_find_files" | "fs_read_file" | "fs_read_summary" | "math_eval" | "python_interpreter" | "retrieval_fuzz_search" | "retrieval_vector_search" | "retrieval_web_search"> type FileMergeHandler = ( filename: string, @@ -210,6 +212,7 @@ interface ScriptRuntimeOptions { * - `system.functions`: use functions * - `system.math`: Math expression evaluator * - `system.python`: Expert at generating and understanding Python code. +* - `system.python_interpreter`: Python Dockerized code execution * - `system.retrieval_fuzz_search`: Full Text Fuzzy Search * - `system.retrieval_vector_search`: Embeddings Vector Search * - `system.retrieval_web_search`: Web Search @@ -230,6 +233,7 @@ interface ScriptRuntimeOptions { * - `fs_read_file`: Reads a file as text from the file system. * - `fs_read_summary`: Reads a summary of a file from the file system. * - `math_eval`: Evaluates a math expression +* - `python_interpreter`: Executes python 3.12 code in a docker container. The process output is returned. Use 'print' to output data. * - `retrieval_fuzz_search`: Search for keywords using the full text of files and a fuzzy distance. * - `retrieval_vector_search`: Search files using embeddings and similarity distance. * - `retrieval_web_search`: Search the web for a user query using Bing Search. @@ -1622,6 +1626,11 @@ interface ContainerHost extends ShellHost { * @param toContainer directory in the container */ copyTo(fromHost: string | string[], toContainer: string): Promise + + /** + * Stops and cleans out the container + */ + stop(): Promise } interface PromptContext extends ChatGenerationContext {