defDiff (#733)

* diff package

* Add createDiff function and tests, extend defDiff options in types

* Add createDefDiff function and refactor diff handling across core modules

* Remove unused logDiff console output and extra newline

* Update investigator report formatting for commit and diff links

packages/cli/package.json

@@ -56,6 +56,7 @@
     "promptfoo": "0.89.3"
   },
   "devDependencies": {
+    "@types/diff": "^5.2.2",
     "@types/dockerode": "^3.3.31",
     "@types/fs-extra": "^11.0.4",
     "@types/memorystream": "^0.3.4",
@@ -65,6 +66,7 @@
     "@types/replace-ext": "^2.0.2",
     "@types/ws": "^8.5.12",
     "commander": "^12.1.0",
+    "diff": "^7.0.0",
     "dotenv": "^16.4.5",
     "esbuild": "^0.24.0",
     "execa": "^9.4.0",

packages/core/package.json

@@ -20,6 +20,7 @@
     "@azure/identity": "^4.4.1",
     "@huggingface/jinja": "^0.3.1",
     "@tidyjs/tidy": "^2.5.2",
+    "@types/diff": "^5.2.2",
     "@types/html-escaper": "^3.0.2",
     "@types/html-to-text": "^9.0.4",
     "@types/inflection": "^1.13.2",

packages/core/src/diff.test.ts

@@ -1,6 +1,6 @@
 import { describe, test } from "node:test"
 import assert from "node:assert/strict"
-import { parseLLMDiffs } from "./diff"
+import { createDiff, parseLLMDiffs } from "./diff"
 
 describe("diff", () => {
     test("is_valid_email", () => {
@@ -287,3 +287,33 @@ describe("diff", () => {
         assert.notEqual(chunks.length, 0)
     })
 })
+test("createDiff with context", () => {
+    const left = {
+        filename: "file1.txt",
+        content: "line1\nline2\nline3\nline4\nline5\n",
+    }
+    const right = {
+        filename: "file1.txt",
+        content: "line1\nline2\nline3\nline4 modified\nline5\n",
+    }
+    const diff = createDiff(left, right, { context: 2 })
+    assert(diff.includes("@@ -2,4 +2,4 @@"))
+    assert(diff.includes("-line4"))
+    assert(diff.includes("+line4 modified"))
+})
+
+test("createDiff without context", () => {
+    const left = {
+        filename: "file1.txt",
+        content: "line1\nline2\nline3\nline4\nline5\n",
+    }
+    const right = {
+        filename: "file1.txt",
+        content: "line1\nline2\nline3\nline4 modified\nline5\n",
+    }
+    const diff = createDiff(left, right)
+    console.log(diff)
+    assert(diff.includes("@@ -1,5 +1,5 @@"))
+    assert(diff.includes("-line4"))
+    assert(diff.includes("+line4 modified"))
+})

packages/core/src/diff.ts

@@ -1,5 +1,6 @@
 import { assert } from "./util"
 import parseDiff from "parse-diff"
+import { createTwoFilesPatch } from "diff"
 
 export interface Chunk {
     state: "existing" | "deleted" | "added"
@@ -283,3 +284,24 @@ export function llmifyDiff(diff: string) {
 
     return result
 }
+
+export function createDiff(
+    left: WorkspaceFile,
+    right: WorkspaceFile,
+    options?: { context?: number }
+) {
+    const res = createTwoFilesPatch(
+        left.filename,
+        right.filename,
+        left.content,
+        right.content,
+        undefined,
+        undefined,
+        {
+            ignoreCase: true,
+            ignoreWhitespace: true,
+            ...(options ?? {}),
+        }
+    )
+    return res.replace(/^[^=]*={10,}\n/, "")
+}