From 2aa65bab515649ef586b87dc36762abb18ff676c Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Tue, 15 Oct 2024 16:50:34 +0000 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20refactor:=20update=20file=20edits?= =?UTF-8?q?=20logic=20and=20fix=20validation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/cli/src/run.ts | 3 +-- packages/core/src/fileedits.ts | 13 +++++++++---- packages/core/src/runpromptcontext.ts | 4 +--- packages/core/src/schema.ts | 2 +- packages/core/src/types/prompt_template.d.ts | 10 +++++----- packages/sample/src/edits/editsgen.genai.mjs | 2 +- packages/sample/src/edits/fib.rb | 2 +- 7 files changed, 19 insertions(+), 17 deletions(-) diff --git a/packages/cli/src/run.ts b/packages/cli/src/run.ts index 7b6a424bca..9fb90bfcab 100644 --- a/packages/cli/src/run.ts +++ b/packages/cli/src/run.ts @@ -367,8 +367,7 @@ export async function runScript( if (isJSONLFilename(outData)) await appendJSONL(outData, result.frames) else await writeText(outData, JSON.stringify(result.frames, null, 2)) - if (result.status === "success" && result.fileEdits && applyEdits) - await writeFileEdits(result.fileEdits, { trace }) + await writeFileEdits(result.fileEdits, { applyEdits, trace }) const promptjson = result.messages?.length ? JSON.stringify(result.messages, null, 2) diff --git a/packages/core/src/fileedits.ts b/packages/core/src/fileedits.ts index d5ecdb8976..e68186b63c 100644 --- a/packages/core/src/fileedits.ts +++ b/packages/core/src/fileedits.ts @@ -282,18 +282,23 @@ function validateFileOutputs( */ export async function writeFileEdits( fileEdits: Record, // Contains the edits to be applied to files - options?: TraceOptions + options?: { applyEdits?: boolean } & TraceOptions ) { - const { trace } = options || {} + const { applyEdits, trace } = options || {} // Iterate over each file edit entry for (const fileEdit of Object.entries(fileEdits || {})) { // Destructure the filename, before content, after content, and validation from the entry const [fn, { before, after, validation }] = fileEdit + if (!applyEdits && !validation?.valid) { + // path not validated + continue + } + // Skip writing if the edit is invalid and applyEdits is false - if (validation?.valid === false) { + if (validation?.error) { trace.detailsFenced( - `skipping ${fn}, invalid`, + `skipping ${fn}, invalid schema`, validation.error, "text" ) diff --git a/packages/core/src/runpromptcontext.ts b/packages/core/src/runpromptcontext.ts index 06a0a88ddb..805aac5180 100644 --- a/packages/core/src/runpromptcontext.ts +++ b/packages/core/src/runpromptcontext.ts @@ -691,9 +691,7 @@ export function createChatGenerationContext( ) ) tracePromptResult(runTrace, resp) - const { fileEdits } = resp - if (fileEdits?.length && applyEdits) - await writeFileEdits(fileEdits, { trace }) + await writeFileEdits(resp.fileEdits, { applyEdits, trace }) return resp } catch (e) { runTrace.error(e) diff --git a/packages/core/src/schema.ts b/packages/core/src/schema.ts index 480a7a41b2..b551f0479c 100644 --- a/packages/core/src/schema.ts +++ b/packages/core/src/schema.ts @@ -168,7 +168,7 @@ export function validateJSONWithSchema( object: any, schema: JSONSchema, options?: { trace: MarkdownTrace } -): JSONSchemaValidation { +): FileEditValidation { const { trace } = options || {} if (!schema) return { diff --git a/packages/core/src/types/prompt_template.d.ts b/packages/core/src/types/prompt_template.d.ts index 3550f82564..e265279114 100644 --- a/packages/core/src/types/prompt_template.d.ts +++ b/packages/core/src/types/prompt_template.d.ts @@ -932,7 +932,7 @@ interface JSONSchemaArray { type JSONSchema = JSONSchemaObject | JSONSchemaArray -interface JSONSchemaValidation { +interface FileEditValidation { schema?: JSONSchema valid: boolean error?: string @@ -941,7 +941,7 @@ interface JSONSchemaValidation { interface DataFrame { schema?: string data: unknown - validation?: JSONSchemaValidation + validation?: FileEditValidation } interface RunPromptResult { @@ -1023,7 +1023,7 @@ interface Fenced { content: string args?: { schema?: string } & Record - validation?: JSONSchemaValidation + validation?: FileEditValidation } interface XMLParseOptions { @@ -1257,7 +1257,7 @@ interface Parsers { * @param schema JSON schema instance * @param content object to validate */ - validateJSON(schema: JSONSchema, content: any): JSONSchemaValidation + validateJSON(schema: JSONSchema, content: any): FileEditValidation /** * Renders a mustache template @@ -2107,7 +2107,7 @@ interface ChatTurnGenerationContext { interface FileUpdate { before: string after: string - validation?: JSONSchemaValidation + validation?: FileEditValidation } interface RunPromptResultPromiseWithOptions extends Promise { diff --git a/packages/sample/src/edits/editsgen.genai.mjs b/packages/sample/src/edits/editsgen.genai.mjs index dcb2d0b6ef..6b45b27d75 100644 --- a/packages/sample/src/edits/editsgen.genai.mjs +++ b/packages/sample/src/edits/editsgen.genai.mjs @@ -1,7 +1,7 @@ script({ files: "src/edits/fib.ts", }) -$`Generation 10 variations of the SNIPPET in various programming languages and save them in files. +$`Generation 1 variations of the SNIPPET in various programming languages and save them in files. - there should be lines with comments - there should be a function with a TODO comment and a BODY comment diff --git a/packages/sample/src/edits/fib.rb b/packages/sample/src/edits/fib.rb index f7b847fc1a..93308636bd 100644 --- a/packages/sample/src/edits/fib.rb +++ b/packages/sample/src/edits/fib.rb @@ -1,5 +1,5 @@ def fibonacci(n) # TODO: implement fibonacci algorithm - 0 # BODY + return 0 # BODY end # this one needs to go # another comment