diff --git a/docs/genaisrc/genaiscript.d.ts b/docs/genaisrc/genaiscript.d.ts index faed3976ba..f1e2965211 100644 --- a/docs/genaisrc/genaiscript.d.ts +++ b/docs/genaisrc/genaiscript.d.ts @@ -1003,6 +1003,9 @@ interface RunPromptResult { | "cancel" | "fail" usages?: ChatCompletionUsages + fileEdits?: Record + edits?: Edits[] + changelogs?: ChangeLog[] } /** @@ -2045,6 +2048,11 @@ interface PromptGeneratorOptions extends ModelOptions, PromptSystemOptions { * Label for trace */ label?: string + + /** + * Write file edits to the file system + */ + applyEdits?: boolean } interface FileOutputOptions { @@ -2172,7 +2180,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { name: string, description: string, parameters: PromptParametersSchema | JSONSchema, - fn: ChatFunctionHandler + fn: ChatFunctionHandler, + options?: DefToolOptions ): void defAgent( name: string, @@ -2197,6 +2206,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { strings: TemplateStringsArray, ...args: any[] ): RunPromptResultPromiseWithOptions + defFileMerge(fn: FileMergeHandler): void + defOutputProcessor(fn: PromptOutputProcessorHandler): void } interface GenerationOutput { @@ -2971,8 +2982,6 @@ interface ContainerHost extends ShellHost { interface PromptContext extends ChatGenerationContext { script(options: PromptArgs): void system(options: PromptSystemArgs): void - defFileMerge(fn: FileMergeHandler): void - defOutputProcessor(fn: PromptOutputProcessorHandler): void env: ExpansionVariables path: Path parsers: Parsers diff --git a/docs/src/content/docs/reference/scripts/system.mdx b/docs/src/content/docs/reference/scripts/system.mdx index b020953551..4437ef699a 100644 --- a/docs/src/content/docs/reference/scripts/system.mdx +++ b/docs/src/content/docs/reference/scripts/system.mdx @@ -206,7 +206,10 @@ defAgent( - The current repository is the same as github repository. - Prefer using diff to compare files rather than listing files. Listing files is only useful when you need to read the content of the files. `, - { model, system: ["system.git_info", "system.github_info"], tools: ["git"] } + { + model, + system: ["system.git_info", "system.github_info", "system.git"], + } ) ````` diff --git a/eval/extrism/genaisrc/extrism-tool.genai.mts b/eval/extrism/genaisrc/extrism-tool.genai.mts deleted file mode 100644 index 5b05f81dff..0000000000 --- a/eval/extrism/genaisrc/extrism-tool.genai.mts +++ /dev/null @@ -1,100 +0,0 @@ -script({ - model: "openai:gpt-4o", - title: "Runs a extrism sample", - system: ["system", "system.explanations", "system.python", "system.files"], -}) - -const practiceDir = "python/exercises/practice" -const { sample = "anagram" } = env.vars - -// create container -const container = await host.container({ - image: "python:3.11.1", - networkEnabled: true, -}) -//console.log(`container path: ${await container.containerPath}`) -console.log(`host path: ${await container.hostPath}`) - -await container.exec("git clone https://github.com/exercism/python") -await container.exec("pip install --upgrade pip") -await container.exec("pip install -r requirements.txt --user", { - cwd: "python", -}) -await container.exec("pip install pytest --user", { cwd: "python" }) -await container.disconnect() - -// generate sample -const cwd = path.join(practiceDir, sample) -const { files: samplefiles } = JSON.parse( - await container.readText(path.join(cwd, ".meta/config.json")) -) -const { solution } = samplefiles -const filename = path.join(cwd, solution[0]) -let instructions = "" -for (const introname of ["introduction", "instructions", "instructions.app"]) { - const intro = await container.readText( - path.join(cwd, `.docs/${introname}.md`) - ) - if (intro) instructions += intro + "\n\n" -} - -$` - -## Task 1: - -Analyze INSTRUCTIONS and generate Python code that the requirements. - -- use Python 3.11 -- Use the TEMPLATE code as a starting point and update it to solve the problem. -You mush NOT change the function signature. Implement the functions. -- do NOT generate unit tests. -- you can only use built-in python libraries. pip packages are not allowed. - -## Task 2: - -Evaluate the generated code by running the unit tests. - -- Use the test_code tool to run the unit tests against the generated code. - -## Task 3: - -If the tests fail, update the generated code in Task 1 and re-run the tests in Task 1. -If the tests passed, stop. -` - -def("INSTRUCTIONS", instructions, { language: "markdown" }) -def("TEMPLATE", { filename }, { language: "python" }) - -let generatedCode = "" -let testPassed = true -defTool( - "test_code", - "Run unit tests against generated solution", - { - code: { - type: "string", - description: "Generated Python code to solve the problem", - }, - }, - async ({ code }) => { - generatedCode = code - console.log(code) - await container.writeText(filename, code) - const res = await container.exec( - `python3.11 -m pytest -o markers=task ${sample}_test.py`, - { cwd } - ) - if (res.exitCode) { - console.log(res.stdout || "") - console.log(res.stderr || "") - } - testPassed = true - return res - } -) - -defOutputProcessor(async (res) => { - if (!generatedCode) throw new Error("Generated code is missing") - if (!testPassed) throw new Error("Unit tests failed") - return { text: res.text + "\n\n" } -}) diff --git a/eval/extrism/genaisrc/extrism.genai.mts b/eval/extrism/genaisrc/extrism.genai.mts new file mode 100644 index 0000000000..0a1fffcffc --- /dev/null +++ b/eval/extrism/genaisrc/extrism.genai.mts @@ -0,0 +1,129 @@ +script({ + model: "openai:gpt-4o", + title: "Runs a extrism sample", +}) + +const results = `eval/extrism/results/${new Date().toISOString().replace(/:/g, "-")}` +const practiceDir = "python/exercises/practice" + +// create container +const container = await host.container({ + image: "python:3.11.1", + networkEnabled: true, +}) +//console.log(`container path: ${await container.containerPath}`) +console.log(`host path: ${await container.hostPath}`) + +await container.exec("git clone https://github.com/exercism/python") +await container.exec("pip install --upgrade pip") +await container.exec("pip install -r requirements.txt --user", { + cwd: "python", +}) +await container.exec("pip install pytest --user", { cwd: "python" }) +await container.disconnect() + +const q = host.promiseQueue(5) +const samples = (await container.exec("ls -1", { cwd: practiceDir })).stdout + .trim() + .split(/\n/g) +await q.mapAll(samples, async (sample) => { + const cwd = path.join(practiceDir, sample) + console.log(cwd) + const { files: samplefiles } = JSON.parse( + await container.readText(path.join(cwd, ".meta/config.json")) + ) + const { solution, test } = samplefiles + const filename = path.join(cwd, solution[0]) + let instructions = "" + for (const introname of [ + "introduction", + "instructions", + "instructions.app", + ]) { + const intro = await container.readText( + path.join(cwd, `.docs/${introname}.md`) + ) + if (intro) instructions += intro + "\n\n" + } + + let generatedCode = "" + let testPassed = false + const res = await runPrompt( + (ctx) => { + ctx.$` + +## Task 1: + +Analyze INSTRUCTIONS and generate Python code that the requirements. + +- use Python 3.11 +- Use the TEMPLATE code as a starting point and update it to solve the problem. +You mush NOT change the function signature. Implement the functions. +- do NOT generate unit tests. +- you can only use built-in python libraries. pip packages are not allowed. + +## Task 2: + +Evaluate the generated code by running the unit tests. + +- Use the test_code tool to run the unit tests against the generated code. + +## Task 3: + +If the tests fail, update the generated code in Task 1 and re-run the tests in Task 1. +If the tests passed, stop. +` + + ctx.def("INSTRUCTIONS", instructions, { language: "markdown" }) + ctx.def("TEMPLATE", { filename }, { language: "python" }) + + ctx.defTool( + "test_code", + "Run unit tests against generated solution", + { + code: { + type: "string", + description: + "Generated Python code to solve the problem", + }, + }, + async ({ code }) => { + generatedCode = code + console.log(code) + await container.writeText(filename, code) + const res = await container.exec( + `python3.11 -m pytest -o markers=task ${test[0]}`, + { cwd } + ) + if (res.exitCode) { + console.log(res.stdout || "") + console.log(res.stderr || "") + } else testPassed = true + return res + }, + { maxTokens: 20000 } + ) + }, + { + label: sample, + applyEdits: true, + cache: "extrism", + system: [ + "system", + "system.explanations", + "system.python", + "system.files", + ], + } + ) + + await workspace.writeText( + `${results}/res/${sample}.yaml`, + YAML.stringify(res) + ) + if (generatedCode) + await workspace.writeText( + `${results}/${testPassed ? "success" : "failed"}/${solution[0]}`, + generatedCode + ) +}) diff --git a/eval/extrism/genaisrc/genaiscript.d.ts b/eval/extrism/genaisrc/genaiscript.d.ts index faed3976ba..f1e2965211 100644 --- a/eval/extrism/genaisrc/genaiscript.d.ts +++ b/eval/extrism/genaisrc/genaiscript.d.ts @@ -1003,6 +1003,9 @@ interface RunPromptResult { | "cancel" | "fail" usages?: ChatCompletionUsages + fileEdits?: Record + edits?: Edits[] + changelogs?: ChangeLog[] } /** @@ -2045,6 +2048,11 @@ interface PromptGeneratorOptions extends ModelOptions, PromptSystemOptions { * Label for trace */ label?: string + + /** + * Write file edits to the file system + */ + applyEdits?: boolean } interface FileOutputOptions { @@ -2172,7 +2180,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { name: string, description: string, parameters: PromptParametersSchema | JSONSchema, - fn: ChatFunctionHandler + fn: ChatFunctionHandler, + options?: DefToolOptions ): void defAgent( name: string, @@ -2197,6 +2206,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { strings: TemplateStringsArray, ...args: any[] ): RunPromptResultPromiseWithOptions + defFileMerge(fn: FileMergeHandler): void + defOutputProcessor(fn: PromptOutputProcessorHandler): void } interface GenerationOutput { @@ -2971,8 +2982,6 @@ interface ContainerHost extends ShellHost { interface PromptContext extends ChatGenerationContext { script(options: PromptArgs): void system(options: PromptSystemArgs): void - defFileMerge(fn: FileMergeHandler): void - defOutputProcessor(fn: PromptOutputProcessorHandler): void env: ExpansionVariables path: Path parsers: Parsers diff --git a/genaisrc/genaiscript.d.ts b/genaisrc/genaiscript.d.ts index faed3976ba..f1e2965211 100644 --- a/genaisrc/genaiscript.d.ts +++ b/genaisrc/genaiscript.d.ts @@ -1003,6 +1003,9 @@ interface RunPromptResult { | "cancel" | "fail" usages?: ChatCompletionUsages + fileEdits?: Record + edits?: Edits[] + changelogs?: ChangeLog[] } /** @@ -2045,6 +2048,11 @@ interface PromptGeneratorOptions extends ModelOptions, PromptSystemOptions { * Label for trace */ label?: string + + /** + * Write file edits to the file system + */ + applyEdits?: boolean } interface FileOutputOptions { @@ -2172,7 +2180,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { name: string, description: string, parameters: PromptParametersSchema | JSONSchema, - fn: ChatFunctionHandler + fn: ChatFunctionHandler, + options?: DefToolOptions ): void defAgent( name: string, @@ -2197,6 +2206,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { strings: TemplateStringsArray, ...args: any[] ): RunPromptResultPromiseWithOptions + defFileMerge(fn: FileMergeHandler): void + defOutputProcessor(fn: PromptOutputProcessorHandler): void } interface GenerationOutput { @@ -2971,8 +2982,6 @@ interface ContainerHost extends ShellHost { interface PromptContext extends ChatGenerationContext { script(options: PromptArgs): void system(options: PromptSystemArgs): void - defFileMerge(fn: FileMergeHandler): void - defOutputProcessor(fn: PromptOutputProcessorHandler): void env: ExpansionVariables path: Path parsers: Parsers diff --git a/packages/auto/genaiscript.d.ts b/packages/auto/genaiscript.d.ts index faed3976ba..f1e2965211 100644 --- a/packages/auto/genaiscript.d.ts +++ b/packages/auto/genaiscript.d.ts @@ -1003,6 +1003,9 @@ interface RunPromptResult { | "cancel" | "fail" usages?: ChatCompletionUsages + fileEdits?: Record + edits?: Edits[] + changelogs?: ChangeLog[] } /** @@ -2045,6 +2048,11 @@ interface PromptGeneratorOptions extends ModelOptions, PromptSystemOptions { * Label for trace */ label?: string + + /** + * Write file edits to the file system + */ + applyEdits?: boolean } interface FileOutputOptions { @@ -2172,7 +2180,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { name: string, description: string, parameters: PromptParametersSchema | JSONSchema, - fn: ChatFunctionHandler + fn: ChatFunctionHandler, + options?: DefToolOptions ): void defAgent( name: string, @@ -2197,6 +2206,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { strings: TemplateStringsArray, ...args: any[] ): RunPromptResultPromiseWithOptions + defFileMerge(fn: FileMergeHandler): void + defOutputProcessor(fn: PromptOutputProcessorHandler): void } interface GenerationOutput { @@ -2971,8 +2982,6 @@ interface ContainerHost extends ShellHost { interface PromptContext extends ChatGenerationContext { script(options: PromptArgs): void system(options: PromptSystemArgs): void - defFileMerge(fn: FileMergeHandler): void - defOutputProcessor(fn: PromptOutputProcessorHandler): void env: ExpansionVariables path: Path parsers: Parsers diff --git a/packages/cli/src/run.ts b/packages/cli/src/run.ts index fe654acfc6..3b9ac677e3 100644 --- a/packages/cli/src/run.ts +++ b/packages/cli/src/run.ts @@ -59,7 +59,7 @@ import { } from "../../core/src/util" import { YAMLStringify } from "../../core/src/yaml" import { PromptScriptRunOptions } from "../../core/src/server/messages" -import { writeFileEdits } from "../../core/src/edits" +import { writeFileEdits } from "../../core/src/fileedits" import { azureDevOpsCreateIssueComment, AzureDevOpsEnv, @@ -355,8 +355,8 @@ export async function runScript( if (isJSONLFilename(outData)) await appendJSONL(outData, result.frames) else await writeText(outData, JSON.stringify(result.frames, null, 2)) - if (result.status === "success" && result.fileEdits) - await writeFileEdits(result, applyEdits) + if (result.status === "success" && result.fileEdits && applyEdits) + await writeFileEdits(result.fileEdits, { trace }) const promptjson = result.messages?.length ? JSON.stringify(result.messages, null, 2) diff --git a/packages/core/src/chat.ts b/packages/core/src/chat.ts index ef3a51611c..953fd9ed7d 100644 --- a/packages/core/src/chat.ts +++ b/packages/core/src/chat.ts @@ -47,6 +47,7 @@ import { fenceMD, prettifyMarkdown } from "./markdown" import { YAMLStringify } from "./yaml" import { resolveTokenEncoder } from "./encoders" import { estimateTokens, truncateTextToTokens } from "./tokens" +import { computeFileEdits } from "./fileedits" export function toChatCompletionUserMessage( expanded: string, @@ -401,16 +402,19 @@ function assistantText( return text } -function structurifyChatSession( +async function structurifyChatSession( messages: ChatCompletionMessageParam[], schemas: Record, genVars: Record, + fileOutputs: FileOutput[], + outputProcessors: PromptOutputProcessorHandler[], + fileMerges: FileMergeHandler[], options: GenerationOptions, others?: { resp?: ChatCompletionResponse err?: any } -): RunPromptResult { +): Promise { const { trace, responseType, responseSchema } = options const { resp, err } = others || {} const text = assistantText(messages, responseType) @@ -458,7 +462,7 @@ function structurifyChatSession( if (fences?.length) frames.push(...validateFencesWithSchema(fences, schemas, { trace })) - return { + const res = { text, annotations, finishReason, @@ -469,6 +473,14 @@ function structurifyChatSession( genVars, schemas, } + await computeFileEdits(res, { + trace, + schemas, + fileOutputs, + fileMerges, + outputProcessors, + }) + return res } async function processChatMessage( @@ -479,6 +491,9 @@ async function processChatMessage( chatParticipants: ChatParticipant[], schemas: Record, genVars: Record, + fileOutputs: FileOutput[], + outputProcessors: PromptOutputProcessorHandler[], + fileMerges: FileMergeHandler[], options: GenerationOptions ): Promise { const { @@ -554,9 +569,18 @@ async function processChatMessage( if (needsNewTurn) return undefined } - return structurifyChatSession(messages, schemas, genVars, options, { - resp, - }) + return structurifyChatSession( + messages, + schemas, + genVars, + fileOutputs, + outputProcessors, + fileMerges, + options, + { + resp, + } + ) } export function mergeGenerationOptions( @@ -585,6 +609,9 @@ export async function executeChatSession( messages: ChatCompletionMessageParam[], toolDefinitions: ToolCallback[], schemas: Record, + fileOutputs: FileOutput[], + outputProcessors: PromptOutputProcessorHandler[], + fileMerges: FileMergeHandler[], completer: ChatCompletionHandler, chatParticipants: ChatParticipant[], genOptions: GenerationOptions @@ -683,6 +710,9 @@ export async function executeChatSession( chatParticipants, schemas, genVars, + fileOutputs, + outputProcessors, + fileMerges, genOptions ) if (output) return output @@ -691,6 +721,9 @@ export async function executeChatSession( messages, schemas, genVars, + fileOutputs, + outputProcessors, + fileMerges, genOptions, { resp, err } ) diff --git a/packages/core/src/edits.ts b/packages/core/src/edits.ts deleted file mode 100644 index 52173d140f..0000000000 --- a/packages/core/src/edits.ts +++ /dev/null @@ -1,37 +0,0 @@ -// This module provides functionality to write file edits to disk, -// supporting conditional application of edits based on validation results. - -import { GenerationResult } from "./generation" // Import type for generation results -import { writeText } from "./fs" // Import function to write text to files -import { logVerbose } from "./util" // Import function for verbose logging - -/** - * Asynchronously writes file edits to disk. - * - * @param res - The result of a generation process containing file edits. - * @param applyEdits - A flag indicating whether edits should be applied even if validation fails. - */ -export async function writeFileEdits( - res: GenerationResult, // Contains the edits to be applied to files - applyEdits: boolean // Determines whether to apply edits unconditionally -) { - // Iterate over each file edit entry - for (const fileEdit of Object.entries(res.fileEdits)) { - // Destructure the filename, before content, after content, and validation from the entry - const [fn, { before, after, validation }] = fileEdit - - // Skip writing if the edit is invalid and applyEdits is false - if (!validation?.valid && !applyEdits) continue - - // Check if there's a change between before and after content - if (after !== before) { - // Log whether the file is being updated or created - logVerbose( - `${before !== undefined ? `updating` : `creating`} ${fn}` - ) - - // Write the new content to the file - await writeText(fn, after ?? before) // Write 'after' content if available, otherwise 'before' - } - } -} diff --git a/packages/core/src/fileedits.ts b/packages/core/src/fileedits.ts new file mode 100644 index 0000000000..876f5dfa32 --- /dev/null +++ b/packages/core/src/fileedits.ts @@ -0,0 +1,307 @@ +import { applyChangeLog, parseChangeLogs } from "./changelog" +import { CSVToMarkdown } from "./csv" +import { applyLLMDiff, applyLLMPatch, createDiff, parseLLMDiffs } from "./diff" +import { errorMessage } from "./error" +import { unquote } from "./fence" +import { fileExists, readText } from "./fs" +import { isGlobMatch } from "./glob" +import { runtimeHost } from "./host" +import { JSON5parse } from "./json5" +import { stringToPos } from "./parser" +import { validateJSONWithSchema } from "./schema" +import { MarkdownTrace, TraceOptions } from "./trace" +import { logError, logVerbose, relativePath } from "./util" +import { YAMLParse } from "./yaml" +import { writeText } from "./fs" + +export async function computeFileEdits( + res: RunPromptResult, + options: TraceOptions & { + fileOutputs: FileOutput[] + schemas?: Record + fileMerges?: FileMergeHandler[] + outputProcessors?: PromptOutputProcessorHandler[] + } +): Promise { + const { trace, fileOutputs, fileMerges, outputProcessors, schemas } = + options || {} + const { fences, frames, genVars } = res + let text = res.text + let annotations = res.annotations?.slice(0) + const fileEdits: Record = {} + const changelogs: string[] = [] + const edits: Edits[] = [] + const projFolder = runtimeHost.projectFolder() + + // Helper function to get or create file edit object + const getFileEdit = async (fn: string) => { + fn = relativePath(projFolder, fn) + let fileEdit = fileEdits[fn] + if (!fileEdit) { + let before: string = null + let after: string = undefined + if (await fileExists(fn)) before = await readText(fn) + else if (await fileExists(fn)) after = await readText(fn) + fileEdit = fileEdits[fn] = { before, after } + } + return fileEdit + } + + for (const fence of fences.filter( + ({ validation }) => validation?.valid !== false + )) { + const { label: name, content: val, language } = fence + const pm = /^((file|diff):?)\s+/i.exec(name) + if (pm) { + const kw = pm[1].toLowerCase() + const n = unquote(name.slice(pm[0].length).trim()) + const fn = /^[^\/]/.test(n) + ? runtimeHost.resolvePath(projFolder, n) + : n + const fileEdit = await getFileEdit(fn) + if (kw === "file") { + if (fileMerges.length) { + try { + for (const fileMerge of fileMerges) + fileEdit.after = + (await fileMerge( + fn, + "", // todo + fileEdit.after ?? fileEdit.before, + val + )) ?? val + } catch (e) { + logVerbose(e) + trace.error(`error custom merging diff in ${fn}`, e) + } + } else fileEdit.after = val + } else if (kw === "diff") { + const chunks = parseLLMDiffs(val) + try { + fileEdit.after = applyLLMPatch( + fileEdit.after || fileEdit.before, + chunks + ) + } catch (e) { + logVerbose(e) + trace.error(`error applying patch to ${fn}`, e) + try { + fileEdit.after = applyLLMDiff( + fileEdit.after || fileEdit.before, + chunks + ) + } catch (e) { + logVerbose(e) + trace.error(`error merging diff in ${fn}`, e) + } + } + } + } else if (/^changelog$/i.test(name) || /^changelog/i.test(language)) { + changelogs.push(val) + const cls = parseChangeLogs(val) + for (const changelog of cls) { + const { filename } = changelog + const fn = /^[^\/]/.test(filename) // TODO + ? runtimeHost.resolvePath(projFolder, filename) + : filename + const fileEdit = await getFileEdit(fn) + fileEdit.after = applyChangeLog( + fileEdit.after || fileEdit.before || "", + changelog + ) + } + } + } + + // Apply user-defined output processors + if (outputProcessors?.length) { + try { + trace.startDetails("🖨️ output processors") + for (const outputProcessor of outputProcessors) { + const { + text: newText, + files, + annotations: oannotations, + } = (await outputProcessor({ + text, + fileEdits, + fences, + frames, + genVars, + annotations, + schemas, + })) || {} + + if (newText !== undefined) { + text = newText + trace.detailsFenced(`📝 text`, text) + } + + if (files) + for (const [n, content] of Object.entries(files)) { + const fn = runtimeHost.path.isAbsolute(n) + ? n + : runtimeHost.resolvePath(projFolder, n) + trace.detailsFenced(`📁 file ${fn}`, content) + const fileEdit = await getFileEdit(fn) + fileEdit.after = content + fileEdit.validation = { valid: true } + } + if (oannotations) annotations = oannotations.slice(0) + } + } catch (e) { + logError(e) + trace.error(`output processor failed`, e) + } finally { + trace.endDetails() + } + } + + // Validate and apply file outputs + validateFileOutputs(fileOutputs, trace, fileEdits, schemas) + + // Convert file edits into structured edits + Object.entries(fileEdits) + .filter(([, { before, after }]) => before !== after) // ignore unchanged files + .forEach(([fn, { before, after, validation }]) => { + if (before) { + edits.push({ + label: `Update ${fn}`, + filename: fn, + type: "replace", + range: [[0, 0], stringToPos(after)], + text: after, + validated: validation?.valid, + }) + } else { + edits.push({ + label: `Create ${fn}`, + filename: fn, + type: "createfile", + text: after, + overwrite: true, + validated: validation?.valid, + }) + } + }) + + if (edits.length) + trace.details( + "✏️ edits", + CSVToMarkdown(edits, { + headers: ["type", "filename", "message", "validated"], + }) + ) + + res.text = text + res.fileEdits = fileEdits + res.changelogs = changelogs + res.annotations = annotations + res.edits = edits +} + +// Validate file outputs against specified schemas and patterns +/** + * Validates file outputs based on provided patterns and schemas. + * @param fileOutputs List of file outputs to validate. + * @param trace The markdown trace for logging. + * @param fileEdits Record of file updates. + * @param schemas The JSON schemas for validation. + */ +function validateFileOutputs( + fileOutputs: FileOutput[], + trace: MarkdownTrace, + fileEdits: Record, + schemas: Record +) { + if (fileOutputs?.length && Object.keys(fileEdits || {}).length) { + trace.startDetails("🗂 file outputs") + for (const fileEditName of Object.keys(fileEdits)) { + const fe = fileEdits[fileEditName] + for (const fileOutput of fileOutputs) { + const { pattern, options } = fileOutput + if (isGlobMatch(fileEditName, pattern)) { + try { + trace.startDetails(`📁 ${fileEditName}`) + trace.itemValue(`pattern`, pattern) + const { schema: schemaId } = options || {} + if (/\.(json|yaml)$/i.test(fileEditName)) { + const { after } = fileEdits[fileEditName] + const data = /\.json$/i.test(fileEditName) + ? JSON5parse(after) + : YAMLParse(after) + trace.detailsFenced("📝 data", data) + if (schemaId) { + const schema = schemas[schemaId] + if (!schema) + fe.validation = { + valid: false, + error: `schema ${schemaId} not found`, + } + else + fe.validation = validateJSONWithSchema( + data, + schema, + { + trace, + } + ) + } + } else { + fe.validation = { valid: true } + } + } catch (e) { + trace.error(errorMessage(e)) + fe.validation = { + valid: false, + error: errorMessage(e), + } + } finally { + trace.endDetails() + } + break + } + } + } + trace.endDetails() + } +} + +/** + * Asynchronously writes file edits to disk. + * + * @param res - The result of a generation process containing file edits. + * @param applyEdits - A flag indicating whether edits should be applied even if validation fails. + */ +export async function writeFileEdits( + fileEdits: Record, // Contains the edits to be applied to files + options?: TraceOptions +) { + const { trace } = options || {} + // Iterate over each file edit entry + for (const fileEdit of Object.entries(fileEdits || {})) { + // Destructure the filename, before content, after content, and validation from the entry + const [fn, { before, after, validation }] = fileEdit + + // Skip writing if the edit is invalid and applyEdits is false + if (!validation?.valid) continue + + // Check if there's a change between before and after content + if (after !== before) { + // Log whether the file is being updated or created + logVerbose( + `${before !== undefined ? `updating` : `creating`} ${fn}` + ) + trace.detailsFenced( + `updating ${fn}`, + createDiff( + { filename: fn, content: before }, + { filename: fn, content: after } + ), + "diff" + ) + // Write the new content to the file + await writeText(fn, after ?? before) // Write 'after' content if available, otherwise 'before' + } + } +} diff --git a/packages/core/src/genaisrc/genaiscript.d.ts b/packages/core/src/genaisrc/genaiscript.d.ts index faed3976ba..f1e2965211 100644 --- a/packages/core/src/genaisrc/genaiscript.d.ts +++ b/packages/core/src/genaisrc/genaiscript.d.ts @@ -1003,6 +1003,9 @@ interface RunPromptResult { | "cancel" | "fail" usages?: ChatCompletionUsages + fileEdits?: Record + edits?: Edits[] + changelogs?: ChangeLog[] } /** @@ -2045,6 +2048,11 @@ interface PromptGeneratorOptions extends ModelOptions, PromptSystemOptions { * Label for trace */ label?: string + + /** + * Write file edits to the file system + */ + applyEdits?: boolean } interface FileOutputOptions { @@ -2172,7 +2180,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { name: string, description: string, parameters: PromptParametersSchema | JSONSchema, - fn: ChatFunctionHandler + fn: ChatFunctionHandler, + options?: DefToolOptions ): void defAgent( name: string, @@ -2197,6 +2206,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { strings: TemplateStringsArray, ...args: any[] ): RunPromptResultPromiseWithOptions + defFileMerge(fn: FileMergeHandler): void + defOutputProcessor(fn: PromptOutputProcessorHandler): void } interface GenerationOutput { @@ -2971,8 +2982,6 @@ interface ContainerHost extends ShellHost { interface PromptContext extends ChatGenerationContext { script(options: PromptArgs): void system(options: PromptSystemArgs): void - defFileMerge(fn: FileMergeHandler): void - defOutputProcessor(fn: PromptOutputProcessorHandler): void env: ExpansionVariables path: Path parsers: Parsers diff --git a/packages/core/src/genaisrc/system.agent_docs.genai.mjs b/packages/core/src/genaisrc/system.agent_docs.genai.mjs index 513a3898ea..4288050780 100644 --- a/packages/core/src/genaisrc/system.agent_docs.genai.mjs +++ b/packages/core/src/genaisrc/system.agent_docs.genai.mjs @@ -30,7 +30,7 @@ defAgent( system: ["system.explanations", "system.github_info"], tools: [ "md_find_files", - "md_read_frontmatterm", + "md_read_frontmatter", "fs_find_files", "fs_read_file", ], diff --git a/packages/core/src/genaisrc/system.agent_git.genai.mjs b/packages/core/src/genaisrc/system.agent_git.genai.mjs index 4304e4703c..35aa448c6d 100644 --- a/packages/core/src/genaisrc/system.agent_git.genai.mjs +++ b/packages/core/src/genaisrc/system.agent_git.genai.mjs @@ -12,5 +12,8 @@ defAgent( - The current repository is the same as github repository. - Prefer using diff to compare files rather than listing files. Listing files is only useful when you need to read the content of the files. `, - { model, system: ["system.git_info", "system.github_info"], tools: ["git"] } + { + model, + system: ["system.git_info", "system.github_info", "system.git"], + } ) diff --git a/packages/core/src/importprompt.ts b/packages/core/src/importprompt.ts index a89eade576..6cbc3b19d3 100644 --- a/packages/core/src/importprompt.ts +++ b/packages/core/src/importprompt.ts @@ -1,6 +1,5 @@ -import { assert } from "console" import { host } from "./host" -import { logError } from "./util" +import { logError, logVerbose } from "./util" import { TraceOptions } from "./trace" import { pathToFileURL } from "url" import { resolveGlobal } from "./globals" @@ -33,10 +32,10 @@ export async function importPrompt( try { // override global context for (const field of Object.keys(ctx0)) { - assert( - field === "console" || leakables.includes(field) || !glb[field], - `overriding global field ${field}` - ) + //logVerbose( + // field === "console" || leakables.includes(field) || !glb[field], + // `overriding global field ${field}` + //) oldGlb[field] = glb[field] glb[field] = (ctx0 as any)[field] } diff --git a/packages/core/src/promptcontext.ts b/packages/core/src/promptcontext.ts index 800b7e5c70..e8cdbc515b 100644 --- a/packages/core/src/promptcontext.ts +++ b/packages/core/src/promptcontext.ts @@ -193,11 +193,6 @@ export async function createPromptContext( }, } - // Default output processor for the prompt - const defOutputProcessor = (fn: PromptOutputProcessorHandler) => { - if (fn) appendPromptChild(createOutputProcessor(fn)) - } - // Define the host for executing commands, browsing, and other operations const promptHost: PromptHost = Object.freeze({ exec: async ( @@ -263,10 +258,6 @@ export async function createPromptContext( parsers, retrieval, host: promptHost, - defOutputProcessor, - defFileMerge: (fn) => { - appendPromptChild(createFileMerge(fn)) - }, } env.generator = ctx ctx.env = Object.freeze(env) diff --git a/packages/core/src/promptrunner.ts b/packages/core/src/promptrunner.ts index 696333c568..9fe895b8f6 100644 --- a/packages/core/src/promptrunner.ts +++ b/packages/core/src/promptrunner.ts @@ -23,7 +23,6 @@ import { validateJSONWithSchema } from "./schema" import { YAMLParse } from "./yaml" import { expandTemplate } from "./expander" import { resolveLanguageModel } from "./lm" -import { Stats } from "fs" // Asynchronously resolve expansion variables needed for a template /** @@ -188,24 +187,6 @@ export async function runTemplate( frames: [], } } - const fileEdits: Record = {} - const changelogs: string[] = [] - const edits: Edits[] = [] - const projFolder = runtimeHost.projectFolder() - - // Helper function to get or create file edit object - const getFileEdit = async (fn: string) => { - fn = relativePath(projFolder, fn) - let fileEdit = fileEdits[fn] - if (!fileEdit) { - let before: string = null - let after: string = undefined - if (await fileExists(fn)) before = await readText(fn) - else if (await fileExists(fn)) after = await readText(fn) - fileEdit = fileEdits[fn] = { before, after } - } - return fileEdit - } // Resolve model connection information const connection = await resolveModelConnectionInfo( @@ -242,6 +223,9 @@ export async function runTemplate( messages, functions, schemas, + fileOutputs, + outputProcessors, + fileMerges, completer, chatParticipants, genOptions @@ -256,166 +240,15 @@ export async function runTemplate( error, finishReason, usages, + fileEdits, + changelogs, + edits, } = output let { text, annotations } = output - // Handle fenced code regions within the output - if (json === undefined) { - for (const fence of fences.filter( - ({ validation }) => validation?.valid !== false - )) { - const { label: name, content: val, language } = fence - const pm = /^((file|diff):?)\s+/i.exec(name) - if (pm) { - const kw = pm[1].toLowerCase() - const n = unquote(name.slice(pm[0].length).trim()) - const fn = /^[^\/]/.test(n) - ? runtimeHost.resolvePath(projFolder, n) - : n - const fileEdit = await getFileEdit(fn) - if (kw === "file") { - if (fileMerges.length) { - try { - for (const fileMerge of fileMerges) - fileEdit.after = - (await fileMerge( - fn, - label, - fileEdit.after ?? fileEdit.before, - val - )) ?? val - } catch (e) { - logVerbose(e) - trace.error( - `error custom merging diff in ${fn}`, - e - ) - } - } else fileEdit.after = val - } else if (kw === "diff") { - const chunks = parseLLMDiffs(val) - try { - fileEdit.after = applyLLMPatch( - fileEdit.after || fileEdit.before, - chunks - ) - } catch (e) { - logVerbose(e) - trace.error(`error applying patch to ${fn}`, e) - try { - fileEdit.after = applyLLMDiff( - fileEdit.after || fileEdit.before, - chunks - ) - } catch (e) { - logVerbose(e) - trace.error(`error merging diff in ${fn}`, e) - } - } - } - } else if ( - /^changelog$/i.test(name) || - /^changelog/i.test(language) - ) { - changelogs.push(val) - const cls = parseChangeLogs(val) - for (const changelog of cls) { - const { filename } = changelog - const fn = /^[^\/]/.test(filename) // TODO - ? runtimeHost.resolvePath(projFolder, filename) - : filename - const fileEdit = await getFileEdit(fn) - fileEdit.after = applyChangeLog( - fileEdit.after || fileEdit.before || "", - changelog - ) - } - } - } - } - - // Apply user-defined output processors - if (outputProcessors?.length) { - try { - trace.startDetails("🖨️ output processors") - for (const outputProcessor of outputProcessors) { - const { - text: newText, - files, - annotations: oannotations, - } = (await outputProcessor({ - text, - fileEdits, - fences, - frames, - genVars, - annotations, - schemas, - })) || {} - - if (newText !== undefined) { - text = newText - trace.detailsFenced(`📝 text`, text) - } - - if (files) - for (const [n, content] of Object.entries(files)) { - const fn = runtimeHost.path.isAbsolute(n) - ? n - : runtimeHost.resolvePath(projFolder, n) - trace.detailsFenced(`📁 file ${fn}`, content) - const fileEdit = await getFileEdit(fn) - fileEdit.after = content - fileEdit.validation = { valid: true } - } - if (oannotations) annotations = oannotations.slice(0) - } - } catch (e) { - logError(e) - trace.error(`output processor failed`, e) - } finally { - trace.endDetails() - } - } - - // Validate and apply file outputs - validateFileOutputs(fileOutputs, trace, fileEdits, schemas) - - // Convert file edits into structured edits - Object.entries(fileEdits) - .filter(([, { before, after }]) => before !== after) // ignore unchanged files - .forEach(([fn, { before, after, validation }]) => { - if (before) { - edits.push({ - label: `Update ${fn}`, - filename: fn, - type: "replace", - range: [[0, 0], stringToPos(after)], - text: after, - validated: validation?.valid, - }) - } else { - edits.push({ - label: `Create ${fn}`, - filename: fn, - type: "createfile", - text: after, - overwrite: true, - validated: validation?.valid, - }) - } - }) - // Reporting and tracing output if (fences?.length) trace.details("📩 code regions", renderFencedVariables(fences)) - if (edits.length) - trace.details( - "✏️ edits", - CSVToMarkdown(edits, { - headers: ["type", "filename", "message", "validated"], - }) - ) if (annotations?.length) trace.details( "⚠️ annotations", diff --git a/packages/core/src/runpromptcontext.ts b/packages/core/src/runpromptcontext.ts index cb99e166dc..54826388ca 100644 --- a/packages/core/src/runpromptcontext.ts +++ b/packages/core/src/runpromptcontext.ts @@ -14,6 +14,8 @@ import { createStringTemplateNode, createTextNode, renderPromptNode, + createOutputProcessor, + createFileMerge, } from "./promptdom" import { MarkdownTrace } from "./trace" import { GenerationOptions } from "./generation" @@ -56,6 +58,7 @@ import { concurrentLimit } from "./concurrency" import { Project } from "./ast" import { dedent } from "./indent" import { runtimeHost } from "./host" +import { writeFileEdits } from "./fileedits" export function createChatTurnGenerationContext( options: GenerationOptions, @@ -242,6 +245,11 @@ export function createChatGenerationContext( const turnCtx = createChatTurnGenerationContext(options, trace) const node = turnCtx.node + // Default output processor for the prompt + const defOutputProcessor = (fn: PromptOutputProcessorHandler) => { + if (fn) appendChild(node, createOutputProcessor(fn)) + } + const defTool: ( name: | string @@ -477,7 +485,7 @@ export function createChatGenerationContext( generator: string | PromptGenerator, runOptions?: PromptGeneratorOptions ): Promise => { - const { label } = runOptions || {} + const { label, applyEdits } = runOptions || {} const runTrace = trace.startTraceDetails(`🎁 run prompt ${label || ""}`) try { infoCb?.({ text: `run prompt ${label || ""}` }) @@ -505,6 +513,9 @@ export function createChatGenerationContext( let tools: ToolCallback[] = undefined let schemas: Record = undefined let chatParticipants: ChatParticipant[] = undefined + const fileMerges: FileMergeHandler[] = [] + const outputProcessors: PromptOutputProcessorHandler[] = [] + const fileOutputs: FileOutput[] = [] // expand template const { provider } = parseModelIdentifier(genOptions.model) @@ -519,6 +530,9 @@ export function createChatGenerationContext( functions: fns, messages: msgs, chatParticipants: cps, + fileMerges: fms, + outputProcessors: ops, + fileOutputs: fos, } = await renderPromptNode(genOptions.model, node, { flexTokens: genOptions.flexTokens, trace: runTrace, @@ -528,6 +542,9 @@ export function createChatGenerationContext( tools = fns chatParticipants = cps messages.push(...msgs) + fileMerges.push(...fms) + outputProcessors.push(...ops) + fileOutputs.push(...fos) if (errors?.length) { logError(errors.map((err) => errorMessage(err)).join("\n")) @@ -564,13 +581,13 @@ export function createChatGenerationContext( if (sysr.schemas) Object.assign(schemas, sysr.schemas) if (sysr.functions) tools.push(...sysr.functions) if (sysr.fileMerges?.length) - throw new NotSupportedError("fileMerges") + fileMerges.push(...sysr.fileMerges) if (sysr.outputProcessors?.length) - throw new NotSupportedError("outputProcessors") + outputProcessors.push(...sysr.outputProcessors) if (sysr.chatParticipants) chatParticipants.push(...sysr.chatParticipants) if (sysr.fileOutputs?.length) - throw new NotSupportedError("fileOutputs") + fileOutputs.push(...sysr.fileOutputs) if (sysr.logs?.length) runTrace.details("📝 console.log", sysr.logs) if (sysr.text) { @@ -624,12 +641,18 @@ export function createChatGenerationContext( messages, tools, schemas, + fileOutputs, + outputProcessors, + fileMerges, completer, chatParticipants, genOptions ) ) tracePromptResult(runTrace, resp) + const { fileEdits } = resp + if (fileEdits?.length && applyEdits) + await writeFileEdits(fileEdits, { trace }) return resp } catch (e) { runTrace.error(e) @@ -643,6 +666,10 @@ export function createChatGenerationContext( } } + const defFileMerge = (fn: FileMergeHandler) => { + appendChild(node, createFileMerge(fn)) + } + const ctx = { ...turnCtx, defAgent, @@ -651,6 +678,8 @@ export function createChatGenerationContext( defImages, defChatParticipant, defFileOutput, + defOutputProcessor, + defFileMerge, prompt, runPrompt, } diff --git a/packages/core/src/types/prompt_template.d.ts b/packages/core/src/types/prompt_template.d.ts index 34678dfba4..f3c3beb21a 100644 --- a/packages/core/src/types/prompt_template.d.ts +++ b/packages/core/src/types/prompt_template.d.ts @@ -922,6 +922,9 @@ interface RunPromptResult { | "cancel" | "fail" usages?: ChatCompletionUsages + fileEdits?: Record + edits?: Edits[] + changelogs?: ChangeLog[] } /** @@ -1964,6 +1967,11 @@ interface PromptGeneratorOptions extends ModelOptions, PromptSystemOptions { * Label for trace */ label?: string + + /** + * Write file edits to the file system + */ + applyEdits?: boolean } interface FileOutputOptions { @@ -2091,7 +2099,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { name: string, description: string, parameters: PromptParametersSchema | JSONSchema, - fn: ChatFunctionHandler + fn: ChatFunctionHandler, + options?: DefToolOptions ): void defAgent( name: string, @@ -2116,6 +2125,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { strings: TemplateStringsArray, ...args: any[] ): RunPromptResultPromiseWithOptions + defFileMerge(fn: FileMergeHandler): void + defOutputProcessor(fn: PromptOutputProcessorHandler): void } interface GenerationOutput { @@ -2890,8 +2901,6 @@ interface ContainerHost extends ShellHost { interface PromptContext extends ChatGenerationContext { script(options: PromptArgs): void system(options: PromptSystemArgs): void - defFileMerge(fn: FileMergeHandler): void - defOutputProcessor(fn: PromptOutputProcessorHandler): void env: ExpansionVariables path: Path parsers: Parsers diff --git a/packages/sample/genaisrc/blog/genaiscript.d.ts b/packages/sample/genaisrc/blog/genaiscript.d.ts index faed3976ba..f1e2965211 100644 --- a/packages/sample/genaisrc/blog/genaiscript.d.ts +++ b/packages/sample/genaisrc/blog/genaiscript.d.ts @@ -1003,6 +1003,9 @@ interface RunPromptResult { | "cancel" | "fail" usages?: ChatCompletionUsages + fileEdits?: Record + edits?: Edits[] + changelogs?: ChangeLog[] } /** @@ -2045,6 +2048,11 @@ interface PromptGeneratorOptions extends ModelOptions, PromptSystemOptions { * Label for trace */ label?: string + + /** + * Write file edits to the file system + */ + applyEdits?: boolean } interface FileOutputOptions { @@ -2172,7 +2180,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { name: string, description: string, parameters: PromptParametersSchema | JSONSchema, - fn: ChatFunctionHandler + fn: ChatFunctionHandler, + options?: DefToolOptions ): void defAgent( name: string, @@ -2197,6 +2206,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { strings: TemplateStringsArray, ...args: any[] ): RunPromptResultPromiseWithOptions + defFileMerge(fn: FileMergeHandler): void + defOutputProcessor(fn: PromptOutputProcessorHandler): void } interface GenerationOutput { @@ -2971,8 +2982,6 @@ interface ContainerHost extends ShellHost { interface PromptContext extends ChatGenerationContext { script(options: PromptArgs): void system(options: PromptSystemArgs): void - defFileMerge(fn: FileMergeHandler): void - defOutputProcessor(fn: PromptOutputProcessorHandler): void env: ExpansionVariables path: Path parsers: Parsers diff --git a/packages/sample/genaisrc/genaiscript.d.ts b/packages/sample/genaisrc/genaiscript.d.ts index faed3976ba..f1e2965211 100644 --- a/packages/sample/genaisrc/genaiscript.d.ts +++ b/packages/sample/genaisrc/genaiscript.d.ts @@ -1003,6 +1003,9 @@ interface RunPromptResult { | "cancel" | "fail" usages?: ChatCompletionUsages + fileEdits?: Record + edits?: Edits[] + changelogs?: ChangeLog[] } /** @@ -2045,6 +2048,11 @@ interface PromptGeneratorOptions extends ModelOptions, PromptSystemOptions { * Label for trace */ label?: string + + /** + * Write file edits to the file system + */ + applyEdits?: boolean } interface FileOutputOptions { @@ -2172,7 +2180,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { name: string, description: string, parameters: PromptParametersSchema | JSONSchema, - fn: ChatFunctionHandler + fn: ChatFunctionHandler, + options?: DefToolOptions ): void defAgent( name: string, @@ -2197,6 +2206,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { strings: TemplateStringsArray, ...args: any[] ): RunPromptResultPromiseWithOptions + defFileMerge(fn: FileMergeHandler): void + defOutputProcessor(fn: PromptOutputProcessorHandler): void } interface GenerationOutput { @@ -2971,8 +2982,6 @@ interface ContainerHost extends ShellHost { interface PromptContext extends ChatGenerationContext { script(options: PromptArgs): void system(options: PromptSystemArgs): void - defFileMerge(fn: FileMergeHandler): void - defOutputProcessor(fn: PromptOutputProcessorHandler): void env: ExpansionVariables path: Path parsers: Parsers diff --git a/packages/sample/genaisrc/github-agent.genai.mts b/packages/sample/genaisrc/github-agent.genai.mts index edb130283b..adb99f1010 100644 --- a/packages/sample/genaisrc/github-agent.genai.mts +++ b/packages/sample/genaisrc/github-agent.genai.mts @@ -15,15 +15,18 @@ script({ }) const { - workflow = "latest failed", + workflow = "build.yml", failure_run_id = "latest", - branch = await git.defaultBranch(), + branch = await git.branch(), } = env.vars -$`Investigate the status of the ${workflow} workflow and identify the root cause of the failure of run ${failure_run_id} in branch ${branch}. - -- Correlate the failure with the relevant commits, pull requests or issues. -- Compare the source code between the failed run and the last successful run before that run. +$` +0. Find the worflow ${workflow} in the repository +1. Find the latest failed run of ${workflow} +2. Find the last successful run before the failed run +3. Compare the run job logs between the failed run and the last successful run +4. Compare the source code between the failed run commit (head_sha) and the last successful run commit (head_sha) +5. Analyze all the above information and identify the root cause of the failure In your report, include html links to the relevant runs, commits, pull requests or issues. ` diff --git a/packages/sample/genaisrc/node/genaiscript.d.ts b/packages/sample/genaisrc/node/genaiscript.d.ts index faed3976ba..f1e2965211 100644 --- a/packages/sample/genaisrc/node/genaiscript.d.ts +++ b/packages/sample/genaisrc/node/genaiscript.d.ts @@ -1003,6 +1003,9 @@ interface RunPromptResult { | "cancel" | "fail" usages?: ChatCompletionUsages + fileEdits?: Record + edits?: Edits[] + changelogs?: ChangeLog[] } /** @@ -2045,6 +2048,11 @@ interface PromptGeneratorOptions extends ModelOptions, PromptSystemOptions { * Label for trace */ label?: string + + /** + * Write file edits to the file system + */ + applyEdits?: boolean } interface FileOutputOptions { @@ -2172,7 +2180,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { name: string, description: string, parameters: PromptParametersSchema | JSONSchema, - fn: ChatFunctionHandler + fn: ChatFunctionHandler, + options?: DefToolOptions ): void defAgent( name: string, @@ -2197,6 +2206,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { strings: TemplateStringsArray, ...args: any[] ): RunPromptResultPromiseWithOptions + defFileMerge(fn: FileMergeHandler): void + defOutputProcessor(fn: PromptOutputProcessorHandler): void } interface GenerationOutput { @@ -2971,8 +2982,6 @@ interface ContainerHost extends ShellHost { interface PromptContext extends ChatGenerationContext { script(options: PromptArgs): void system(options: PromptSystemArgs): void - defFileMerge(fn: FileMergeHandler): void - defOutputProcessor(fn: PromptOutputProcessorHandler): void env: ExpansionVariables path: Path parsers: Parsers diff --git a/packages/sample/genaisrc/python/genaiscript.d.ts b/packages/sample/genaisrc/python/genaiscript.d.ts index faed3976ba..f1e2965211 100644 --- a/packages/sample/genaisrc/python/genaiscript.d.ts +++ b/packages/sample/genaisrc/python/genaiscript.d.ts @@ -1003,6 +1003,9 @@ interface RunPromptResult { | "cancel" | "fail" usages?: ChatCompletionUsages + fileEdits?: Record + edits?: Edits[] + changelogs?: ChangeLog[] } /** @@ -2045,6 +2048,11 @@ interface PromptGeneratorOptions extends ModelOptions, PromptSystemOptions { * Label for trace */ label?: string + + /** + * Write file edits to the file system + */ + applyEdits?: boolean } interface FileOutputOptions { @@ -2172,7 +2180,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { name: string, description: string, parameters: PromptParametersSchema | JSONSchema, - fn: ChatFunctionHandler + fn: ChatFunctionHandler, + options?: DefToolOptions ): void defAgent( name: string, @@ -2197,6 +2206,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { strings: TemplateStringsArray, ...args: any[] ): RunPromptResultPromiseWithOptions + defFileMerge(fn: FileMergeHandler): void + defOutputProcessor(fn: PromptOutputProcessorHandler): void } interface GenerationOutput { @@ -2971,8 +2982,6 @@ interface ContainerHost extends ShellHost { interface PromptContext extends ChatGenerationContext { script(options: PromptArgs): void system(options: PromptSystemArgs): void - defFileMerge(fn: FileMergeHandler): void - defOutputProcessor(fn: PromptOutputProcessorHandler): void env: ExpansionVariables path: Path parsers: Parsers diff --git a/packages/sample/genaisrc/style/genaiscript.d.ts b/packages/sample/genaisrc/style/genaiscript.d.ts index faed3976ba..f1e2965211 100644 --- a/packages/sample/genaisrc/style/genaiscript.d.ts +++ b/packages/sample/genaisrc/style/genaiscript.d.ts @@ -1003,6 +1003,9 @@ interface RunPromptResult { | "cancel" | "fail" usages?: ChatCompletionUsages + fileEdits?: Record + edits?: Edits[] + changelogs?: ChangeLog[] } /** @@ -2045,6 +2048,11 @@ interface PromptGeneratorOptions extends ModelOptions, PromptSystemOptions { * Label for trace */ label?: string + + /** + * Write file edits to the file system + */ + applyEdits?: boolean } interface FileOutputOptions { @@ -2172,7 +2180,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { name: string, description: string, parameters: PromptParametersSchema | JSONSchema, - fn: ChatFunctionHandler + fn: ChatFunctionHandler, + options?: DefToolOptions ): void defAgent( name: string, @@ -2197,6 +2206,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { strings: TemplateStringsArray, ...args: any[] ): RunPromptResultPromiseWithOptions + defFileMerge(fn: FileMergeHandler): void + defOutputProcessor(fn: PromptOutputProcessorHandler): void } interface GenerationOutput { @@ -2971,8 +2982,6 @@ interface ContainerHost extends ShellHost { interface PromptContext extends ChatGenerationContext { script(options: PromptArgs): void system(options: PromptSystemArgs): void - defFileMerge(fn: FileMergeHandler): void - defOutputProcessor(fn: PromptOutputProcessorHandler): void env: ExpansionVariables path: Path parsers: Parsers diff --git a/packages/sample/genaisrc/style/runprompt.genai.js b/packages/sample/genaisrc/style/runprompt.genai.js index 1d8346b902..fd18f02b8f 100644 --- a/packages/sample/genaisrc/style/runprompt.genai.js +++ b/packages/sample/genaisrc/style/runprompt.genai.js @@ -1,5 +1,5 @@ script({ - model: "openai:gpt-3.5-turbo", + model: "openai:gpt-4o-mini", tests: {}, }) @@ -39,3 +39,28 @@ fence(resPoem.text) $`Is this JSON? Respond yes or no.` fence(resJSON.text) + +const { text, fileEdits } = await runPrompt( + (_) => { + _.$`Create a file named "test.txt" with the following content: "hello world"` + _.defOutputProcessor((output) => { + console.log(`processing output: ${output.text}`) + return { text: output.text + "\n" } + }) + _.defFileMerge((filename, label, before, after) => { + console.log({ filename, label, before, after }) + if (path.basename(filename) !== "test.txt") + throw new Error("wrong file name") + return after + "\n" + }) + }, + { applyEdits: true, label: "outputs", system: ["system", "system.files"] } +) +console.log(text) +console.log(YAML.stringify(fileEdits)) +if (!fileEdits?.["test.txt"]?.after?.includes("hello world")) + throw new Error("File not created") +if (!text.includes("")) + throw new Error("output processor did not run") +if (!fileEdits?.["test.txt"]?.after?.includes("")) + throw new Error("file merge did not run") diff --git a/packages/sample/src/aici/genaiscript.d.ts b/packages/sample/src/aici/genaiscript.d.ts index faed3976ba..f1e2965211 100644 --- a/packages/sample/src/aici/genaiscript.d.ts +++ b/packages/sample/src/aici/genaiscript.d.ts @@ -1003,6 +1003,9 @@ interface RunPromptResult { | "cancel" | "fail" usages?: ChatCompletionUsages + fileEdits?: Record + edits?: Edits[] + changelogs?: ChangeLog[] } /** @@ -2045,6 +2048,11 @@ interface PromptGeneratorOptions extends ModelOptions, PromptSystemOptions { * Label for trace */ label?: string + + /** + * Write file edits to the file system + */ + applyEdits?: boolean } interface FileOutputOptions { @@ -2172,7 +2180,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { name: string, description: string, parameters: PromptParametersSchema | JSONSchema, - fn: ChatFunctionHandler + fn: ChatFunctionHandler, + options?: DefToolOptions ): void defAgent( name: string, @@ -2197,6 +2206,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { strings: TemplateStringsArray, ...args: any[] ): RunPromptResultPromiseWithOptions + defFileMerge(fn: FileMergeHandler): void + defOutputProcessor(fn: PromptOutputProcessorHandler): void } interface GenerationOutput { @@ -2971,8 +2982,6 @@ interface ContainerHost extends ShellHost { interface PromptContext extends ChatGenerationContext { script(options: PromptArgs): void system(options: PromptSystemArgs): void - defFileMerge(fn: FileMergeHandler): void - defOutputProcessor(fn: PromptOutputProcessorHandler): void env: ExpansionVariables path: Path parsers: Parsers diff --git a/packages/sample/src/errors/genaiscript.d.ts b/packages/sample/src/errors/genaiscript.d.ts index faed3976ba..f1e2965211 100644 --- a/packages/sample/src/errors/genaiscript.d.ts +++ b/packages/sample/src/errors/genaiscript.d.ts @@ -1003,6 +1003,9 @@ interface RunPromptResult { | "cancel" | "fail" usages?: ChatCompletionUsages + fileEdits?: Record + edits?: Edits[] + changelogs?: ChangeLog[] } /** @@ -2045,6 +2048,11 @@ interface PromptGeneratorOptions extends ModelOptions, PromptSystemOptions { * Label for trace */ label?: string + + /** + * Write file edits to the file system + */ + applyEdits?: boolean } interface FileOutputOptions { @@ -2172,7 +2180,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { name: string, description: string, parameters: PromptParametersSchema | JSONSchema, - fn: ChatFunctionHandler + fn: ChatFunctionHandler, + options?: DefToolOptions ): void defAgent( name: string, @@ -2197,6 +2206,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { strings: TemplateStringsArray, ...args: any[] ): RunPromptResultPromiseWithOptions + defFileMerge(fn: FileMergeHandler): void + defOutputProcessor(fn: PromptOutputProcessorHandler): void } interface GenerationOutput { @@ -2971,8 +2982,6 @@ interface ContainerHost extends ShellHost { interface PromptContext extends ChatGenerationContext { script(options: PromptArgs): void system(options: PromptSystemArgs): void - defFileMerge(fn: FileMergeHandler): void - defOutputProcessor(fn: PromptOutputProcessorHandler): void env: ExpansionVariables path: Path parsers: Parsers diff --git a/packages/sample/src/genaiscript.d.ts b/packages/sample/src/genaiscript.d.ts index faed3976ba..f1e2965211 100644 --- a/packages/sample/src/genaiscript.d.ts +++ b/packages/sample/src/genaiscript.d.ts @@ -1003,6 +1003,9 @@ interface RunPromptResult { | "cancel" | "fail" usages?: ChatCompletionUsages + fileEdits?: Record + edits?: Edits[] + changelogs?: ChangeLog[] } /** @@ -2045,6 +2048,11 @@ interface PromptGeneratorOptions extends ModelOptions, PromptSystemOptions { * Label for trace */ label?: string + + /** + * Write file edits to the file system + */ + applyEdits?: boolean } interface FileOutputOptions { @@ -2172,7 +2180,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { name: string, description: string, parameters: PromptParametersSchema | JSONSchema, - fn: ChatFunctionHandler + fn: ChatFunctionHandler, + options?: DefToolOptions ): void defAgent( name: string, @@ -2197,6 +2206,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { strings: TemplateStringsArray, ...args: any[] ): RunPromptResultPromiseWithOptions + defFileMerge(fn: FileMergeHandler): void + defOutputProcessor(fn: PromptOutputProcessorHandler): void } interface GenerationOutput { @@ -2971,8 +2982,6 @@ interface ContainerHost extends ShellHost { interface PromptContext extends ChatGenerationContext { script(options: PromptArgs): void system(options: PromptSystemArgs): void - defFileMerge(fn: FileMergeHandler): void - defOutputProcessor(fn: PromptOutputProcessorHandler): void env: ExpansionVariables path: Path parsers: Parsers diff --git a/packages/sample/src/makecode/genaiscript.d.ts b/packages/sample/src/makecode/genaiscript.d.ts index faed3976ba..f1e2965211 100644 --- a/packages/sample/src/makecode/genaiscript.d.ts +++ b/packages/sample/src/makecode/genaiscript.d.ts @@ -1003,6 +1003,9 @@ interface RunPromptResult { | "cancel" | "fail" usages?: ChatCompletionUsages + fileEdits?: Record + edits?: Edits[] + changelogs?: ChangeLog[] } /** @@ -2045,6 +2048,11 @@ interface PromptGeneratorOptions extends ModelOptions, PromptSystemOptions { * Label for trace */ label?: string + + /** + * Write file edits to the file system + */ + applyEdits?: boolean } interface FileOutputOptions { @@ -2172,7 +2180,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { name: string, description: string, parameters: PromptParametersSchema | JSONSchema, - fn: ChatFunctionHandler + fn: ChatFunctionHandler, + options?: DefToolOptions ): void defAgent( name: string, @@ -2197,6 +2206,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { strings: TemplateStringsArray, ...args: any[] ): RunPromptResultPromiseWithOptions + defFileMerge(fn: FileMergeHandler): void + defOutputProcessor(fn: PromptOutputProcessorHandler): void } interface GenerationOutput { @@ -2971,8 +2982,6 @@ interface ContainerHost extends ShellHost { interface PromptContext extends ChatGenerationContext { script(options: PromptArgs): void system(options: PromptSystemArgs): void - defFileMerge(fn: FileMergeHandler): void - defOutputProcessor(fn: PromptOutputProcessorHandler): void env: ExpansionVariables path: Path parsers: Parsers diff --git a/packages/sample/src/tla/genaiscript.d.ts b/packages/sample/src/tla/genaiscript.d.ts index faed3976ba..f1e2965211 100644 --- a/packages/sample/src/tla/genaiscript.d.ts +++ b/packages/sample/src/tla/genaiscript.d.ts @@ -1003,6 +1003,9 @@ interface RunPromptResult { | "cancel" | "fail" usages?: ChatCompletionUsages + fileEdits?: Record + edits?: Edits[] + changelogs?: ChangeLog[] } /** @@ -2045,6 +2048,11 @@ interface PromptGeneratorOptions extends ModelOptions, PromptSystemOptions { * Label for trace */ label?: string + + /** + * Write file edits to the file system + */ + applyEdits?: boolean } interface FileOutputOptions { @@ -2172,7 +2180,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { name: string, description: string, parameters: PromptParametersSchema | JSONSchema, - fn: ChatFunctionHandler + fn: ChatFunctionHandler, + options?: DefToolOptions ): void defAgent( name: string, @@ -2197,6 +2206,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { strings: TemplateStringsArray, ...args: any[] ): RunPromptResultPromiseWithOptions + defFileMerge(fn: FileMergeHandler): void + defOutputProcessor(fn: PromptOutputProcessorHandler): void } interface GenerationOutput { @@ -2971,8 +2982,6 @@ interface ContainerHost extends ShellHost { interface PromptContext extends ChatGenerationContext { script(options: PromptArgs): void system(options: PromptSystemArgs): void - defFileMerge(fn: FileMergeHandler): void - defOutputProcessor(fn: PromptOutputProcessorHandler): void env: ExpansionVariables path: Path parsers: Parsers diff --git a/packages/sample/src/vision/genaiscript.d.ts b/packages/sample/src/vision/genaiscript.d.ts index faed3976ba..f1e2965211 100644 --- a/packages/sample/src/vision/genaiscript.d.ts +++ b/packages/sample/src/vision/genaiscript.d.ts @@ -1003,6 +1003,9 @@ interface RunPromptResult { | "cancel" | "fail" usages?: ChatCompletionUsages + fileEdits?: Record + edits?: Edits[] + changelogs?: ChangeLog[] } /** @@ -2045,6 +2048,11 @@ interface PromptGeneratorOptions extends ModelOptions, PromptSystemOptions { * Label for trace */ label?: string + + /** + * Write file edits to the file system + */ + applyEdits?: boolean } interface FileOutputOptions { @@ -2172,7 +2180,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { name: string, description: string, parameters: PromptParametersSchema | JSONSchema, - fn: ChatFunctionHandler + fn: ChatFunctionHandler, + options?: DefToolOptions ): void defAgent( name: string, @@ -2197,6 +2206,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { strings: TemplateStringsArray, ...args: any[] ): RunPromptResultPromiseWithOptions + defFileMerge(fn: FileMergeHandler): void + defOutputProcessor(fn: PromptOutputProcessorHandler): void } interface GenerationOutput { @@ -2971,8 +2982,6 @@ interface ContainerHost extends ShellHost { interface PromptContext extends ChatGenerationContext { script(options: PromptArgs): void system(options: PromptSystemArgs): void - defFileMerge(fn: FileMergeHandler): void - defOutputProcessor(fn: PromptOutputProcessorHandler): void env: ExpansionVariables path: Path parsers: Parsers diff --git a/packages/vscode/genaisrc/genaiscript.d.ts b/packages/vscode/genaisrc/genaiscript.d.ts index faed3976ba..f1e2965211 100644 --- a/packages/vscode/genaisrc/genaiscript.d.ts +++ b/packages/vscode/genaisrc/genaiscript.d.ts @@ -1003,6 +1003,9 @@ interface RunPromptResult { | "cancel" | "fail" usages?: ChatCompletionUsages + fileEdits?: Record + edits?: Edits[] + changelogs?: ChangeLog[] } /** @@ -2045,6 +2048,11 @@ interface PromptGeneratorOptions extends ModelOptions, PromptSystemOptions { * Label for trace */ label?: string + + /** + * Write file edits to the file system + */ + applyEdits?: boolean } interface FileOutputOptions { @@ -2172,7 +2180,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { name: string, description: string, parameters: PromptParametersSchema | JSONSchema, - fn: ChatFunctionHandler + fn: ChatFunctionHandler, + options?: DefToolOptions ): void defAgent( name: string, @@ -2197,6 +2206,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { strings: TemplateStringsArray, ...args: any[] ): RunPromptResultPromiseWithOptions + defFileMerge(fn: FileMergeHandler): void + defOutputProcessor(fn: PromptOutputProcessorHandler): void } interface GenerationOutput { @@ -2971,8 +2982,6 @@ interface ContainerHost extends ShellHost { interface PromptContext extends ChatGenerationContext { script(options: PromptArgs): void system(options: PromptSystemArgs): void - defFileMerge(fn: FileMergeHandler): void - defOutputProcessor(fn: PromptOutputProcessorHandler): void env: ExpansionVariables path: Path parsers: Parsers diff --git a/slides/genaisrc/genaiscript.d.ts b/slides/genaisrc/genaiscript.d.ts index faed3976ba..f1e2965211 100644 --- a/slides/genaisrc/genaiscript.d.ts +++ b/slides/genaisrc/genaiscript.d.ts @@ -1003,6 +1003,9 @@ interface RunPromptResult { | "cancel" | "fail" usages?: ChatCompletionUsages + fileEdits?: Record + edits?: Edits[] + changelogs?: ChangeLog[] } /** @@ -2045,6 +2048,11 @@ interface PromptGeneratorOptions extends ModelOptions, PromptSystemOptions { * Label for trace */ label?: string + + /** + * Write file edits to the file system + */ + applyEdits?: boolean } interface FileOutputOptions { @@ -2172,7 +2180,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { name: string, description: string, parameters: PromptParametersSchema | JSONSchema, - fn: ChatFunctionHandler + fn: ChatFunctionHandler, + options?: DefToolOptions ): void defAgent( name: string, @@ -2197,6 +2206,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { strings: TemplateStringsArray, ...args: any[] ): RunPromptResultPromiseWithOptions + defFileMerge(fn: FileMergeHandler): void + defOutputProcessor(fn: PromptOutputProcessorHandler): void } interface GenerationOutput { @@ -2971,8 +2982,6 @@ interface ContainerHost extends ShellHost { interface PromptContext extends ChatGenerationContext { script(options: PromptArgs): void system(options: PromptSystemArgs): void - defFileMerge(fn: FileMergeHandler): void - defOutputProcessor(fn: PromptOutputProcessorHandler): void env: ExpansionVariables path: Path parsers: Parsers