jq support in defData

THIRD_PARTY_LICENSES.md

@@ -7091,6 +7091,30 @@ THIS SOFTWARE.
 
 -----------
 
+The following npm package may be included in this product:
+
+ - [email protected]
+
+This package contains the following license:
+
+ISC License
+
+Copyright (c) 2020, Stefan Terdell
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+-----------
+
 The following npm package may be included in this product:
 
  - [email protected]

packages/cli/package.json

@@ -48,7 +48,6 @@
     "@anthropic-ai/bedrock-sdk": "^0.12.0",
     "@anthropic-ai/sdk": "^0.33.1",
     "@azure/identity": "^4.5.0",
-    "@elastic/micro-jq": "^1.8.0",
     "@inquirer/prompts": "^7.2.1",
     "@modelcontextprotocol/sdk": "^1.0.4",
     "@octokit/plugin-paginate-rest": "^11.3.6",
@@ -117,9 +116,9 @@
   "scripts": {
     "compile:runtime": "tsc src/runtime.ts --skipLibCheck --outDir built --declaration --target es2020 --moduleResolution node --module esnext && mv built/runtime.js built/runtime.mjs",
     "compile:api": "esbuild src/api.ts --outfile=built/api.mjs",
-    "compile:cli": "esbuild src/main.ts --metafile=./esbuild.meta.json --bundle --platform=node --target=node20 --outfile=built/genaiscript.cjs --external:tsx --external:esbuild --external:get-tsconfig --external:resolve-pkg-maps --external:dockerode --external:pdfjs-dist --external:web-tree-sitter --external:tree-sitter-wasms --external:promptfoo --external:typescript --external:@lvce-editor/ripgrep --external:gpt-3-encoder --external:mammoth --external:xlsx --external:mathjs --external:@azure/identity --external:gpt-tokenizer --external:playwright --external:@inquirer/prompts --external:jimp --external:turndown --external:turndown-plugin-gfm --external:vectra --external:tabletojson --external:html-to-text --external:@octokit/rest --external:@octokit/plugin-throttling --external:@octokit/plugin-retry --external:@octokit/plugin-paginate-rest --external:skia-canvas --external:@huggingface/transformers --external:@modelcontextprotocol/sdk --external:@anthropic-ai/sdk --external:@anthropic-ai/bedrock-sdk --external:es-toolkit --external:zod --external:zod-to-json-schema --external:@elastic/micro-jq && node ../../scripts/patch-cli.mjs",
+    "compile:cli": "esbuild src/main.ts --metafile=./esbuild.meta.json --bundle --platform=node --target=node20 --outfile=built/genaiscript.cjs --external:tsx --external:esbuild --external:get-tsconfig --external:resolve-pkg-maps --external:dockerode --external:pdfjs-dist --external:web-tree-sitter --external:tree-sitter-wasms --external:promptfoo --external:typescript --external:@lvce-editor/ripgrep --external:gpt-3-encoder --external:mammoth --external:xlsx --external:mathjs --external:@azure/identity --external:gpt-tokenizer --external:playwright --external:@inquirer/prompts --external:jimp --external:turndown --external:turndown-plugin-gfm --external:vectra --external:tabletojson --external:html-to-text --external:@octokit/rest --external:@octokit/plugin-throttling --external:@octokit/plugin-retry --external:@octokit/plugin-paginate-rest --external:skia-canvas --external:@huggingface/transformers --external:@modelcontextprotocol/sdk --external:@anthropic-ai/sdk --external:@anthropic-ai/bedrock-sdk --external:es-toolkit --external:zod --external:zod-to-json-schema && node ../../scripts/patch-cli.mjs",
     "compile": "yarn compile:api && yarn compile:runtime && yarn compile:cli",
-    "compile-debug": "esbuild src/main.ts --sourcemap --metafile=./esbuild.meta.json --bundle --platform=node --target=node20 --outfile=built/genaiscript.cjs --external:tsx --external:esbuild --external:get-tsconfig --external:resolve-pkg-maps --external:dockerode --external:pdfjs-dist --external:web-tree-sitter --external:tree-sitter-wasms --external:promptfoo --external:typescript --external:@lvce-editor/ripgrep --external:gpt-3-encoder --external:mammoth --external:xlsx --external:mathjs --external:@azure/identity --external:gpt-tokenizer --external:playwright --external:@inquirer/prompts --external:jimp --external:turndown --external:turndown-plugin-gfm --external:vectra --external:tabletojson --external:html-to-text --external:@octokit/rest --external:@octokit/plugin-throttling --external:@octokit/plugin-retry --external:@octokit/plugin-paginate-rest --external:skia-canvas --external:@huggingface/transformers --external:@modelcontextprotocol/sdk --external:@anthropic-ai/sdk --external:@anthropic-ai/bedrock-sdk --external:es-toolkit --external:zod --external:zod-to-json-schema --external:@elastic/micro-jq",
+    "compile-debug": "esbuild src/main.ts --sourcemap --metafile=./esbuild.meta.json --bundle --platform=node --target=node20 --outfile=built/genaiscript.cjs --external:tsx --external:esbuild --external:get-tsconfig --external:resolve-pkg-maps --external:dockerode --external:pdfjs-dist --external:web-tree-sitter --external:tree-sitter-wasms --external:promptfoo --external:typescript --external:@lvce-editor/ripgrep --external:gpt-3-encoder --external:mammoth --external:xlsx --external:mathjs --external:@azure/identity --external:gpt-tokenizer --external:playwright --external:@inquirer/prompts --external:jimp --external:turndown --external:turndown-plugin-gfm --external:vectra --external:tabletojson --external:html-to-text --external:@octokit/rest --external:@octokit/plugin-throttling --external:@octokit/plugin-retry --external:@octokit/plugin-paginate-rest --external:skia-canvas --external:@huggingface/transformers --external:@modelcontextprotocol/sdk --external:@anthropic-ai/sdk --external:@anthropic-ai/bedrock-sdk --external:es-toolkit --external:zod --external:zod-to-json-schema",
     "postcompile": "node built/genaiscript.cjs info help > ../../docs/src/content/docs/reference/cli/commands.md",
     "vis:treemap": "npx --yes esbuild-visualizer --metadata esbuild.meta.json --filename esbuild.treemap.html",
     "vis:network": "npx --yes esbuild-visualizer --metadata esbuild.meta.json --filename esbuild.network.html --template network",

packages/core/package.json

@@ -61,6 +61,7 @@
     "inflection": "^3.0.0",
     "ini": "^5.0.0",
     "jimp": "^1.6.0",
+    "jqts": "^0.0.8",
     "json5": "^2.2.3",
     "jsonrepair": "^3.11.2",
     "magic-string": "^0.30.17",

packages/core/src/jq.test.ts

@@ -0,0 +1,45 @@
+import { jq } from "./jq"
+import { describe, test } from "node:test"
+import assert from "node:assert/strict"
+
+describe("jq", () => {
+    test("returns undefined when input is undefined", () => {
+        const result = jq(undefined, ".")
+        assert.strictEqual(result, undefined)
+    })
+
+    test("applies JQ transformation to input data", () => {
+        const input = { name: "John", age: 30 }
+        const query = ".name"
+        const result = jq(input, query)
+        assert.strictEqual(result, "John")
+    })
+
+    test("handles nested objects correctly", () => {
+        const input = { person: { name: "John", age: 30 } }
+        const query = ".person.name"
+        const result = jq(input, query)
+        assert.strictEqual(result, "John")
+    })
+
+    test("returns null for non-existent keys", () => {
+        const input = { name: "John", age: 30 }
+        const query = ".address"
+        const result = jq(input, query)
+        assert.strictEqual(result, null)
+    })
+
+    test("handles arrays correctly", () => {
+        const input = { people: [{ name: "John" }, { name: "Jane" }] }
+        const query = ".people[1].name"
+        const result = jq(input, query)
+        assert.strictEqual(result, "Jane")
+    })
+
+    test("returns entire input when query is '.'", () => {
+        const input = { name: "John", age: 30 }
+        const query = "."
+        const result = jq(input, query)
+        assert.deepStrictEqual(result, input)
+    })
+})

packages/core/src/jq.ts

@@ -0,0 +1,13 @@
+import _jq from "jqts"
+
+/**
+ * Loads and applies JQ transformation to the input data
+ * @param input
+ */
+export function jq(input: any, query: string): any {
+    if (input === undefined) return input
+
+    const pattern = _jq.compile(query)
+    const res = pattern.evaluate(input)[0]
+    return res
+}

packages/core/src/promptdom.ts

@@ -42,6 +42,7 @@ import { runtimeHost } from "./host"
 import { hash } from "./crypto"
 import { startMcpServer } from "./mcp"
 import { tryZodToJsonSchema } from "./zod"
+import { jq } from "./jq"
 
 // Definition of the PromptNode interface which is an essential part of the code structure.
 export interface PromptNode extends ContextExpansionOptions {
@@ -57,6 +58,7 @@ export interface PromptNode extends ContextExpansionOptions {
         | "assistant"
         | "system"
         | "def"
+        | "defData"
         | "chatParticipant"
         | "fileOutput"
         | "importTemplate"
@@ -98,6 +100,13 @@ export interface PromptDefNode extends PromptNode, DefOptions {
     resolved?: WorkspaceFile // Resolved file content
 }
 
+export interface PromptDefDataNode extends PromptNode, DefDataOptions {
+    type: "defData"
+    name: string // Name of the definition
+    value: Awaitable<object | object[]> // Data associated with the definition
+    resolved?: object | object[]
+}
+
 export interface PromptPrediction {
     type: "content"
     content: string
@@ -317,6 +326,48 @@ function renderDefNode(def: PromptDefNode): string {
     return res
 }
 
+function renderDefDataNode(n: PromptDefDataNode): string {
+    const { name, headers, priority, ephemeral, query } = n
+    let data = n.resolved
+    let format = n.format
+    const cacheControl = n.cacheControl ?? (ephemeral ? "ephemeral" : undefined)
+    if (
+        !format &&
+        Array.isArray(data) &&
+        data.length &&
+        (headers?.length || haveSameKeysAndSimpleValues(data))
+    )
+        format = "csv"
+    else if (!format) format = "yaml"
+
+    if (Array.isArray(data)) data = tidyData(data as object[], n)
+    if (query) data = jq(data, query)
+
+    let text: string
+    let lang: string
+    if (Array.isArray(data) && format === "csv") {
+        text = CSVToMarkdown(data)
+    } else if (format === "json") {
+        text = JSON.stringify(data)
+        lang = "json"
+    } else {
+        text = YAMLStringify(data)
+        lang = "yaml"
+    }
+
+    const value = lang
+        ? `${name}:
+\`\`\`${lang}
+${trimNewlines(text)}
+\`\`\`
+`
+        : `${name}:
+${trimNewlines(text)}
+`
+    // TODO maxTokens does not work well with data
+    return value
+}
+
 // Function to create an assistant node.
 export function createAssistantNode(
     value: Awaitable<string>,
@@ -468,50 +519,16 @@ function haveSameKeysAndSimpleValues(data: object[]): boolean {
 // Function to create a text node with data.
 export function createDefData(
     name: string,
-    data: object | object[],
+    value: Awaitable<object | object[]>,
     options?: DefDataOptions
-) {
-    if (data === undefined) return undefined
-    let { format, headers, priority, cacheControl } = options || {}
-    cacheControl =
-        cacheControl ?? (options?.ephemeral ? "ephemeral" : undefined)
-    if (
-        !format &&
-        Array.isArray(data) &&
-        data.length &&
-        (headers?.length || haveSameKeysAndSimpleValues(data))
-    )
-        format = "csv"
-    else if (!format) format = "yaml"
-
-    if (Array.isArray(data)) data = tidyData(data as object[], options)
-
-    let text: string
-    let lang: string
-    if (Array.isArray(data) && format === "csv") {
-        text = CSVToMarkdown(data)
-    } else if (format === "json") {
-        text = JSON.stringify(data)
-        lang = "json"
-    } else {
-        text = YAMLStringify(data)
-        lang = "yaml"
+): PromptDefDataNode {
+    if (value === undefined) return undefined
+    return {
+        type: "defData",
+        name,
+        value,
+        ...(options || {}),
     }
-
-    const value = lang
-        ? `${name}:
-\`\`\`${lang}
-${trimNewlines(text)}
-\`\`\`
-`
-        : `${name}:
-${trimNewlines(text)}
-`
-    // TODO maxTokens does not work well with data
-    return createTextNode(value, {
-        priority,
-        ephemeral: cacheControl === "ephemeral",
-    })
 }
 
 // Function to append a child node to a parent node.
@@ -529,6 +546,7 @@ export interface PromptNodeVisitor {
     afterNode?: (node: PromptNode) => Awaitable<void> // Post node visitor
     text?: (node: PromptTextNode) => Awaitable<void> // Text node visitor
     def?: (node: PromptDefNode) => Awaitable<void> // Definition node visitor
+    defData?: (node: PromptDefDataNode) => Awaitable<void> // Definition data node visitor
     image?: (node: PromptImageNode) => Awaitable<void> // Image node visitor
     schema?: (node: PromptSchemaNode) => Awaitable<void> // Schema node visitor
     tool?: (node: PromptToolNode) => Awaitable<void> // Function node visitor
@@ -553,6 +571,9 @@ export async function visitNode(node: PromptNode, visitor: PromptNodeVisitor) {
         case "def":
             await visitor.def?.(node as PromptDefNode)
             break
+        case "defData":
+            await visitor.defData?.(node as PromptDefDataNode)
+            break
         case "image":
             await visitor.image?.(node as PromptImageNode)
             break
@@ -666,6 +687,19 @@ async function resolvePromptNode(
                 n.error = e
             }
         },
+        defData: async (n) => {
+            try {
+                names.add(n.name)
+                const value = await n.value
+                n.resolved = value
+                const rendered = renderDefDataNode(n)
+                n.preview = rendered
+                n.tokens = estimateTokens(rendered, encoder)
+                n.children = [createTextNode(rendered)]
+            } catch (e) {
+                n.error = e
+            }
+        },
         system: async (n) => {
             try {
                 const value = await n.value

packages/core/src/types/prompt_template.d.ts

@@ -2411,6 +2411,11 @@ interface DefDataOptions
      * Output format in the prompt. Defaults to Markdown table rendering.
      */
     format?: "json" | "yaml" | "csv"
+
+    /**
+     * jq query to filter the data
+     */
+    query?: string
 }
 
 interface DefSchemaOptions {
@@ -2560,7 +2565,7 @@ interface ChatTurnGenerationContext {
     ): string
     defData(
         name: string,
-        data: object[] | object,
+        data: Awaitable<object[] | object>,
         options?: DefDataOptions
     ): string
     defDiff<T extends string | WorkspaceFile>(

packages/core/src/types/prompt_type.d.ts

@@ -267,7 +267,7 @@ declare function defImages(
  */
 declare function defData(
     name: string,
-    data: object[] | object,
+    data: Awaitable<object[] | object>,
     options?: DefDataOptions
 ): string
 

packages/sample/genaisrc/defdata.genai.mjs

@@ -21,7 +21,8 @@ defData(
 $`
 E:
 ${[{ a: 7, b: 8 },
-    { a: 9, b: 10 },]}
+    { a: 9, b: 10 },]}`
+defData("F", [{ a: 1, b: 2 }, { a: 3, b: 4 }], { query: "map({a})"})
 
-Identify the data formats of A,B,C,D,E and return the format results as CSV with key value pairs variable,format.
+$`Identify the data formats of A,B,C,D,E,F and return the format results as CSV with key value pairs variable,format.
 `