✨ refactor: update file edits logic and fix validation

microsoft · Oct 15, 2024 · 2aa65ba · 2aa65ba
1 parent 738dfda
commit 2aa65ba
Show file tree

Hide file tree

Showing 7 changed files with 19 additions and 17 deletions.
diff --git a/packages/cli/src/run.ts b/packages/cli/src/run.ts
@@ -367,8 +367,7 @@ export async function runScript(
         if (isJSONLFilename(outData)) await appendJSONL(outData, result.frames)
         else await writeText(outData, JSON.stringify(result.frames, null, 2))
 
-    if (result.status === "success" && result.fileEdits && applyEdits)
-        await writeFileEdits(result.fileEdits, { trace })
+    await writeFileEdits(result.fileEdits, { applyEdits, trace })
 
     const promptjson = result.messages?.length
         ? JSON.stringify(result.messages, null, 2)

diff --git a/packages/core/src/fileedits.ts b/packages/core/src/fileedits.ts
@@ -282,18 +282,23 @@ function validateFileOutputs(
  */
 export async function writeFileEdits(
     fileEdits: Record<string, FileUpdate>, // Contains the edits to be applied to files
-    options?: TraceOptions
+    options?: { applyEdits?: boolean } & TraceOptions
 ) {
-    const { trace } = options || {}
+    const { applyEdits, trace } = options || {}
     // Iterate over each file edit entry
     for (const fileEdit of Object.entries(fileEdits || {})) {
         // Destructure the filename, before content, after content, and validation from the entry
         const [fn, { before, after, validation }] = fileEdit
 
+        if (!applyEdits && !validation?.valid) {
+            // path not validated
+            continue
+        }
+
         // Skip writing if the edit is invalid and applyEdits is false
-        if (validation?.valid === false) {
+        if (validation?.error) {
             trace.detailsFenced(
-                `skipping ${fn}, invalid`,
+                `skipping ${fn}, invalid schema`,
                 validation.error,
                 "text"
             )

diff --git a/packages/core/src/runpromptcontext.ts b/packages/core/src/runpromptcontext.ts
@@ -691,9 +691,7 @@ export function createChatGenerationContext(
                 )
             )
             tracePromptResult(runTrace, resp)
-            const { fileEdits } = resp
-            if (fileEdits?.length && applyEdits)
-                await writeFileEdits(fileEdits, { trace })
+            await writeFileEdits(resp.fileEdits, { applyEdits, trace })
             return resp
         } catch (e) {
             runTrace.error(e)

diff --git a/packages/core/src/schema.ts b/packages/core/src/schema.ts
@@ -168,7 +168,7 @@ export function validateJSONWithSchema(
     object: any,
     schema: JSONSchema,
     options?: { trace: MarkdownTrace }
-): JSONSchemaValidation {
+): FileEditValidation {
     const { trace } = options || {}
     if (!schema)
         return {

diff --git a/packages/core/src/types/prompt_template.d.ts b/packages/core/src/types/prompt_template.d.ts
@@ -932,7 +932,7 @@ interface JSONSchemaArray {
 
 type JSONSchema = JSONSchemaObject | JSONSchemaArray
 
-interface JSONSchemaValidation {
+interface FileEditValidation {
     schema?: JSONSchema
     valid: boolean
     error?: string
@@ -941,7 +941,7 @@ interface JSONSchemaValidation {
 interface DataFrame {
     schema?: string
     data: unknown
-    validation?: JSONSchemaValidation
+    validation?: FileEditValidation
 }
 
 interface RunPromptResult {
@@ -1023,7 +1023,7 @@ interface Fenced {
     content: string
     args?: { schema?: string } & Record<string, string>
 
-    validation?: JSONSchemaValidation
+    validation?: FileEditValidation
 }
 
 interface XMLParseOptions {
@@ -1257,7 +1257,7 @@ interface Parsers {
      * @param schema JSON schema instance
      * @param content object to validate
      */
-    validateJSON(schema: JSONSchema, content: any): JSONSchemaValidation
+    validateJSON(schema: JSONSchema, content: any): FileEditValidation
 
     /**
      * Renders a mustache template
@@ -2107,7 +2107,7 @@ interface ChatTurnGenerationContext {
 interface FileUpdate {
     before: string
     after: string
-    validation?: JSONSchemaValidation
+    validation?: FileEditValidation
 }
 
 interface RunPromptResultPromiseWithOptions extends Promise<RunPromptResult> {

diff --git a/packages/sample/src/edits/editsgen.genai.mjs b/packages/sample/src/edits/editsgen.genai.mjs
@@ -1,7 +1,7 @@
 script({
     files: "src/edits/fib.ts",
 })
-$`Generation 10 variations of the SNIPPET in various programming languages and save them in files.
+$`Generation 1 variations of the SNIPPET in various programming languages and save them in files.
 
 - there should be lines with comments
 - there should be a function with a TODO comment and a BODY comment

diff --git a/packages/sample/src/edits/fib.rb b/packages/sample/src/edits/fib.rb
@@ -1,5 +1,5 @@
 def fibonacci(n)
     # TODO: implement fibonacci algorithm
-    0 # BODY
+    return 0 # BODY
 end # this one needs to go
 # another comment