moving to groq

microsoft · Dec 23, 2024 · db59ddd · db59ddd
1 parent 33788a1
commit db59ddd
Show file tree

Hide file tree

Showing 14 changed files with 1,309 additions and 114 deletions.
diff --git a/docs/src/content/docs/reference/scripts/parsers.md b/docs/src/content/docs/reference/scripts/parsers.md
@@ -322,12 +322,14 @@ used with `defData`.
 const d = parsers.tidyData(rows, { sliceSample: 100, sort: "name" })
 ```
 
-## jq
+## GROQ
 
-Apply a [jq](https://jqlang.github.io/jq/) query to a JSON object.
+Apply a [GROQ](GROQEvaluate) query to a JSON object.
 
 ```js
-const d = parsers.jq(rows, "map({ a })")
+const d = parsers.GROQ(`*[completed == true && userId == 2]{
+  title
+}`, data)
 ```
 
 ## hash

diff --git a/docs/src/content/docs/reference/scripts/system.mdx b/docs/src/content/docs/reference/scripts/system.mdx
@@ -2555,7 +2555,8 @@ Plain text output
 `````js wrap title="system.output_plaintext"
 system({ title: "Plain text output" })
 $`## Plain Text Output
-Respond in plain text. Do not wrap result in Markdown code fences or XML tags.
+Respond in plain text. No yapping, no markdown, no code fences, no XML tags, no string delimiters
+wrapping it.
 `
 
 `````

diff --git a/docs/yarn.lock b/docs/yarn.lock
@@ -4799,9 +4799,9 @@ sprintf-js@~1.0.2:
   integrity sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==
 
 starlight-blog@^0.16.0:
-  version "0.16.0"
-  resolved "https://registry.yarnpkg.com/starlight-blog/-/starlight-blog-0.16.0.tgz#7954997cb1119305f143b8d391cc8a098b7e09d2"
-  integrity sha512-gskN6cs12Rycya49DXnqK0q9FefNjbojA3QO5w+XugcRZDCcQ7fev8Zs0QDtLoHjibqRhFTrvhurlNe/0T2x+Q==
+  version "0.16.1"
+  resolved "https://registry.yarnpkg.com/starlight-blog/-/starlight-blog-0.16.1.tgz#fbf9da70c678c66e1629c8df1d02a478480cc17c"
+  integrity sha512-9WMpRZHhfgWjf2oQ1oUqGCJOUB3z+JohHrrQxVtzwm9GPAxNeu7/DAx6eQJf3moLbkdzCNP2VyrT2Molo62vRw==
   dependencies:
     "@astrojs/mdx" "^4.0.2"
     "@astrojs/rss" "^4.0.10"

diff --git a/packages/core/package.json b/packages/core/package.json
@@ -54,14 +54,14 @@
     "fflate": "^0.8.2",
     "file-type": "19.1.1",
     "gpt-tokenizer": "^2.8.1",
+    "groq-js": "^1.14.2",
     "html-escaper": "^3.0.3",
     "html-to-text": "^9.0.5",
     "https-proxy-agent": "^7.0.6",
     "ignore": "^6.0.2",
     "inflection": "^3.0.0",
     "ini": "^5.0.0",
     "jimp": "^1.6.0",
-    "jqts": "^0.0.8",
     "json5": "^2.2.3",
     "jsonrepair": "^3.11.2",
     "magic-string": "^0.30.17",

diff --git a/packages/core/src/genaisrc/system.output_plaintext.genai.mjs b/packages/core/src/genaisrc/system.output_plaintext.genai.mjs
@@ -1,4 +1,5 @@
 system({ title: "Plain text output" })
 $`## Plain Text Output
-Respond in plain text. Do not wrap result in Markdown code fences or XML tags.
+Respond in plain text. No yapping, no markdown, no code fences, no XML tags, no string delimiters
+wrapping it.
 `
diff --git a/packages/core/src/groq.ts b/packages/core/src/groq.ts
@@ -0,0 +1,12 @@
+import { parse, evaluate } from "groq-js"
+/**
+ * Loads and applies JQ transformation to the input data
+ * @param input
+ */
+export function GROQEvaluate(query: string, dataset: any): any {
+    if (dataset === undefined) return dataset
+
+    const tree = parse(query)
+    const res = evaluate(tree, { dataset })
+    return res
+}
diff --git a/packages/core/src/jq.test.ts b/packages/core/src/jq.test.ts
diff --git a/packages/core/src/jq.ts b/packages/core/src/jq.ts
diff --git a/packages/core/src/parsers.ts b/packages/core/src/parsers.ts
@@ -28,7 +28,7 @@ import { jinjaRender } from "./jinja"
 import { createDiff, llmifyDiff } from "./diff"
 import { tidyData } from "./tidy"
 import { hash } from "./crypto"
-import { jq } from "./jq"
+import { GROQEvaluate } from "./groq"
 
 export async function createParsers(options: {
     trace: MarkdownTrace
@@ -123,6 +123,6 @@ export async function createParsers(options: {
         tidyData: (rows, options) => tidyData(rows, options),
         hash: async (text, options) => await hash(text, options),
         unfence: unfence,
-        jq: jq,
+        GROQ: GROQEvaluate,
     })
 }
diff --git a/packages/core/src/promptdom.ts b/packages/core/src/promptdom.ts
@@ -42,7 +42,7 @@ import { runtimeHost } from "./host"
 import { hash } from "./crypto"
 import { startMcpServer } from "./mcp"
 import { tryZodToJsonSchema } from "./zod"
-import { jq } from "./jq"
+import { GROQEvaluate } from "./groq"
 
 // Definition of the PromptNode interface which is an essential part of the code structure.
 export interface PromptNode extends ContextExpansionOptions {
@@ -341,7 +341,7 @@ function renderDefDataNode(n: PromptDefDataNode): string {
     else if (!format) format = "yaml"
 
     if (Array.isArray(data)) data = tidyData(data as object[], n)
-    if (query) data = jq(data, query)
+    if (query) data = GROQEvaluate(query, data)
 
     let text: string
     let lang: string

diff --git a/packages/core/src/types/prompt_template.d.ts b/packages/core/src/types/prompt_template.d.ts
@@ -1591,11 +1591,12 @@ interface Parsers {
     tidyData(rows: object[], options?: DataFilter): object[]
 
     /**
-     * Applies a jq query to the data
+     * Applies a GROQ query to the data
      * @param data data object to filter
-     * @param query jq query
+     * @param query query
+     * @see https://groq.dev/
      */
-    jq(data: any, query: string): any
+    GROQ(query: string, data: any): any
 
     /**
      * Computes a sha1 that can be used for hashing purpose, not cryptographic.
@@ -2422,7 +2423,8 @@ interface DefDataOptions
     format?: "json" | "yaml" | "csv"
 
     /**
-     * jq query to filter the data
+     * GROQ query to filter the data
+     * @see https://groq.dev/
      */
     query?: string
 }