JSON5 helper (#964)

* feat: ✨ add JSON5 parsing and stringifying utilities * handle errors in agent * fix duplicate git_diff tool
microsoft · Dec 22, 2024 · 8a92fd8 · 8a92fd8
1 parent 69a412b
commit 8a92fd8
Show file tree

Hide file tree

Showing 21 changed files with 106 additions and 139 deletions.
diff --git a/docs/src/components/BuiltinAgents.mdx b/docs/src/components/BuiltinAgents.mdx
@@ -13,4 +13,4 @@ import { LinkCard } from '@astrojs/starlight/components';
 <LinkCard title="agent interpreter" description="run code interpreters for Python, Math. Use this agent to ground computation questions." href="/genaiscript/reference/scripts/system#systemagent_interpreter" />
 <LinkCard title="agent planner" description="generates a plan to solve a task" href="/genaiscript/reference/scripts/system#systemagent_planner" />
 <LinkCard title="agent user_input" description="ask user for input to confirm, select or answer the question in the query. The message should be very clear and provide all the context." href="/genaiscript/reference/scripts/system#systemagent_user_input" />
-<LinkCard title="agent web-search" description="search the web to accomplish tasks." href="/genaiscript/reference/scripts/system#systemagent_web" />
+<LinkCard title="agent web" description="search the web to accomplish tasks." href="/genaiscript/reference/scripts/system#systemagent_web" />
diff --git a/docs/src/components/BuiltinTools.mdx b/docs/src/components/BuiltinTools.mdx
@@ -13,7 +13,6 @@ import { LinkCard } from '@astrojs/starlight/components';
 <LinkCard title="git_branch_default" description="Gets the default branch using git." href="/genaiscript/reference/scripts/system#systemgit" />
 <LinkCard title="git_branch_current" description="Gets the current branch using git." href="/genaiscript/reference/scripts/system#systemgit" />
 <LinkCard title="git_branch_list" description="List all branches using git." href="/genaiscript/reference/scripts/system#systemgit" />
-<LinkCard title="git_diff" description="Computes file diffs using the git diff command. If the diff is too large, it returns the list of modified/added files." href="/genaiscript/reference/scripts/system#systemgit" />
 <LinkCard title="git_list_commits" description="Generates a history of commits using the git log command." href="/genaiscript/reference/scripts/system#systemgit" />
 <LinkCard title="git_status" description="Generates a status of the repository using git." href="/genaiscript/reference/scripts/system#systemgit" />
 <LinkCard title="git_last_tag" description="Gets the last tag using git." href="/genaiscript/reference/scripts/system#systemgit" />

diff --git a/docs/src/content/docs/reference/scripts/system.mdx b/docs/src/content/docs/reference/scripts/system.mdx
@@ -390,7 +390,7 @@ system({
 const model = env.vars.agentWebSearchModel
 
 defAgent(
-    "web-search",
+    "web",
     "search the web to accomplish tasks.",
     `Your are a helpful LLM agent that can use web search.
     Answer the question in QUERY.`,
@@ -424,8 +424,7 @@ system({
     lineNumbers: true,
 })
 
-$`## Annotation Format
-
+$`## Annotations Format
 Use the following format to report **file annotations** (same as GitHub Actions workflow).
 
 ::(notice|warning|error) file=<filename>,line=<start line>,endLine=<end line>,code=<error_id>::<message>
@@ -552,7 +551,8 @@ system({
     title: "Generate diagrams"
 })
 
-$`Use mermaid syntax if you need to generate state diagrams, class inheritance diagrams, relationships.`
+$`## Diagrams Format
+Use mermaid syntax if you need to generate state diagrams, class inheritance diagrams, relationships.`
 `````
 
 
@@ -1080,7 +1080,6 @@ Tools to query a git repository.
 -  tool `git_branch_default`: Gets the default branch using git.
 -  tool `git_branch_current`: Gets the current branch using git.
 -  tool `git_branch_list`: List all branches using git.
--  tool `git_diff`: Computes file diffs using the git diff command. If the diff is too large, it returns the list of modified/added files.
 -  tool `git_list_commits`: Generates a history of commits using the git log command.
 -  tool `git_status`: Generates a status of the repository using git.
 -  tool `git_last_tag`: Gets the last tag using git.
@@ -1113,58 +1112,6 @@ defTool("git_branch_list", "List all branches using git.", {}, async () => {
     return await git.exec("branch")
 })
 
-defTool(
-    "git_diff",
-    "Computes file diffs using the git diff command. If the diff is too large, it returns the list of modified/added files.",
-    {
-        type: "object",
-        properties: {
-            base: {
-                type: "string",
-                description: "Base branch, ref, commit sha to compare against.",
-            },
-            head: {
-                type: "string",
-                description:
-                    "Head branch, ref, commit sha to compare. Use 'HEAD' to compare against the current branch.",
-            },
-            staged: {
-                type: "boolean",
-                description: "Compare staged changes",
-            },
-            nameOnly: {
-                type: "boolean",
-                description: "Show only file names",
-            },
-            paths: {
-                type: "array",
-                description: "Paths to compare",
-                items: {
-                    type: "string",
-                    description: "File path or wildcard supported by git",
-                },
-            },
-            excludedPaths: {
-                type: "array",
-                description: "Paths to exclude",
-                items: {
-                    type: "string",
-                    description: "File path or wildcard supported by git",
-                },
-            },
-        },
-    },
-    async (args) => {
-        const { context, ...rest } = args
-        const res = await git.diff({
-            llmify: true,
-            ...rest,
-        })
-        return res
-    },
-    { maxTokens: 20000 }
-)
-
 defTool(
     "git_list_commits",
     "Generates a history of commits using the git log command.",

diff --git a/package.json b/package.json
@@ -13,6 +13,7 @@
         "setup": "git submodule update --init --recursive",
         "setup:az": "curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash",
         "setup:bicep": "az bicep upgrade",
+        "az:login": "az login --scope api://trapi/.default",
         "install:playwright": "yarn install playwright --with-deps",
         "install:force": "rm yarn.lock && yarn install && yarn --cwd docs install:force && yarn --cwd slides install:force && yarn gen:licenses",
         "compile-ext": "yarn --cwd packages/core run prompts:bundle && yarn --cwd packages/vscode run compile",

diff --git a/packages/cli/src/nodehost.ts b/packages/cli/src/nodehost.ts
@@ -131,7 +131,8 @@ export class NodeHost implements RuntimeHost {
         if (typeof value === "string") value = { model: value, source }
         const aliases = this._modelAliases[source]
         const c = aliases[id] || (aliases[id] = { source })
-        if (value.model !== undefined) (c as any).model = value.model
+        if (value.model !== undefined && value.model !== id)
+            (c as any).model = value.model
         if (!isNaN(value.temperature))
             (c as any).temperature = value.temperature
     }

diff --git a/packages/core/src/anthropic.ts b/packages/core/src/anthropic.ts
@@ -433,7 +433,7 @@ const completerFactory = (
 
         trace.appendContent("\n\n")
         trace.itemValue(`🏁 finish reason`, finishReason)
-        if (usage) {
+        if (usage?.total_tokens) {
             trace.itemValue(
                 `🪙 tokens`,
                 `${usage.total_tokens} total, ${usage.prompt_tokens} prompt, ${usage.completion_tokens} completion`

diff --git a/packages/core/src/chat.ts b/packages/core/src/chat.ts
@@ -79,6 +79,7 @@ import {
     serializeLogProb,
     topLogprobsToMarkdown,
 } from "./logprob"
+import { uniq } from "es-toolkit"
 
 export function toChatCompletionUserMessage(
     expanded: string,
@@ -892,8 +893,15 @@ export async function executeChatSession(
 
     try {
         trace.startDetails(`🧠 llm chat`)
-        if (toolDefinitions?.length)
+        if (toolDefinitions?.length) {
             trace.detailsFenced(`🛠️ tools`, tools, "yaml")
+            const toolNames = toolDefinitions.map(({ spec }) => spec.name)
+            const duplicates = uniq(toolNames).filter(
+                (name, index) => toolNames.lastIndexOf(name) !== index
+            )
+            if (duplicates.length)
+                throw new Error(`duplicate tools: ${duplicates.join(", ")}`)
+        }
         let genVars: Record<string, string>
         while (true) {
             stats.turns++

diff --git a/packages/core/src/genaisrc/system.agent_web.genai.mjs b/packages/core/src/genaisrc/system.agent_web.genai.mjs
@@ -5,7 +5,7 @@ system({
 const model = env.vars.agentWebSearchModel
 
 defAgent(
-    "web-search",
+    "web",
     "search the web to accomplish tasks.",
     `Your are a helpful LLM agent that can use web search.
     Answer the question in QUERY.`,

diff --git a/packages/core/src/genaisrc/system.annotations.genai.mjs b/packages/core/src/genaisrc/system.annotations.genai.mjs
@@ -5,8 +5,7 @@ system({
     lineNumbers: true,
 })
 
-$`## Annotation Format
-
+$`## Annotations Format
 Use the following format to report **file annotations** (same as GitHub Actions workflow).
 
 ::(notice|warning|error) file=<filename>,line=<start line>,endLine=<end line>,code=<error_id>::<message>

diff --git a/packages/core/src/genaisrc/system.diagrams.genai.mjs b/packages/core/src/genaisrc/system.diagrams.genai.mjs
@@ -2,4 +2,5 @@ system({
     title: "Generate diagrams"
 })
 
-$`Use mermaid syntax if you need to generate state diagrams, class inheritance diagrams, relationships.`
+$`## Diagrams Format
+Use mermaid syntax if you need to generate state diagrams, class inheritance diagrams, relationships.`
diff --git a/packages/core/src/genaisrc/system.git.genai.mjs b/packages/core/src/genaisrc/system.git.genai.mjs
@@ -25,58 +25,6 @@ defTool("git_branch_list", "List all branches using git.", {}, async () => {
     return await git.exec("branch")
 })
 
-defTool(
-    "git_diff",
-    "Computes file diffs using the git diff command. If the diff is too large, it returns the list of modified/added files.",
-    {
-        type: "object",
-        properties: {
-            base: {
-                type: "string",
-                description: "Base branch, ref, commit sha to compare against.",
-            },
-            head: {
-                type: "string",
-                description:
-                    "Head branch, ref, commit sha to compare. Use 'HEAD' to compare against the current branch.",
-            },
-            staged: {
-                type: "boolean",
-                description: "Compare staged changes",
-            },
-            nameOnly: {
-                type: "boolean",
-                description: "Show only file names",
-            },
-            paths: {
-                type: "array",
-                description: "Paths to compare",
-                items: {
-                    type: "string",
-                    description: "File path or wildcard supported by git",
-                },
-            },
-            excludedPaths: {
-                type: "array",
-                description: "Paths to exclude",
-                items: {
-                    type: "string",
-                    description: "File path or wildcard supported by git",
-                },
-            },
-        },
-    },
-    async (args) => {
-        const { context, ...rest } = args
-        const res = await git.diff({
-            llmify: true,
-            ...rest,
-        })
-        return res
-    },
-    { maxTokens: 20000 }
-)
-
 defTool(
     "git_list_commits",
     "Generates a history of commits using the git log command.",

diff --git a/packages/core/src/globals.ts b/packages/core/src/globals.ts
@@ -17,6 +17,7 @@ import { GitClient } from "./git"
 import { estimateTokens, truncateTextToTokens } from "./tokens"
 import { chunk, resolveTokenEncoder } from "./encoders"
 import { runtimeHost } from "./host"
+import { JSON5Stringify, JSON5TryParse } from "./json5"
 
 /**
  * This file defines global utilities and installs them into the global context.
@@ -86,6 +87,11 @@ export function installGlobals() {
         stringify: JSONLStringify, // Convert objects to JSONL string
     })
 
+    glb.JSON5 = Object.freeze<JSON5>({
+        parse: JSON5TryParse,
+        stringify: JSON5Stringify
+    })
+
     // Freeze AICI utilities with a generation function
     glb.AICI = Object.freeze<AICI>({
         gen: (options: AICIGenOptions) => {

diff --git a/packages/core/src/llms.json b/packages/core/src/llms.json
@@ -71,10 +71,10 @@
             "prediction": false,
             "bearerToken": true,
             "aliases": {
-                "large": "gemini-2.0-flash-exp",
+                "large": "gemini-1.5-flash-latest",
                 "small": "gemini-1.5-flash-latest",
-                "vision": "gemini-2.0-flash-exp",
-                "long": "gemini-2.0-flash-exp",
+                "vision": "gemini-1.5-flash-latest",
+                "long": "gemini-1.5-flash-latest",
                 "reasoning": "gemini-2.0-flash-thinking-exp-1219",
                 "reasoning_small": "gemini-2.0-flash-thinking-exp-1219",
                 "embeddings": "text-embedding-004"

diff --git a/packages/core/src/llms.ts b/packages/core/src/llms.ts
@@ -1,3 +1,4 @@
+import { Model } from "@anthropic-ai/sdk/resources/index.mjs"
 import { LARGE_MODEL_ID, SMALL_MODEL_ID, VISION_MODEL_ID } from "./constants"
 import { ModelConfiguration, ModelConfigurations } from "./host"
 import LLMS from "./llms.json"
@@ -14,8 +15,19 @@ export function defaultModelConfigurations(): ModelConfigurations {
     ]
     const res = {
         ...(Object.fromEntries(
-            aliases.map((alias) => [alias, readModelAlias(alias)])
+            aliases.map<[string, ModelConfiguration]>((alias) => [
+                alias,
+                readModelAlias(alias),
+            ])
         ) as ModelConfigurations),
+        ...Object.fromEntries(
+            Object.entries(LLMS.aliases).map<[string, ModelConfiguration]>(
+                ([id, model]) => [
+                    id,
+                    { model, source: "default" } satisfies ModelConfiguration,
+                ]
+            )
+        ),
     }
     return structuredClone(res)
 
@@ -30,6 +42,6 @@ export function defaultModelConfigurations(): ModelConfigurations {
             model: candidates[0],
             candidates,
             source: "default",
-        })
+        } satisfies ModelConfiguration)
     }
 }
diff --git a/packages/core/src/openai.ts b/packages/core/src/openai.ts
@@ -399,7 +399,7 @@ export const OpenAIChatCompletion: ChatCompletionHandler = async (
     trace.appendContent("\n\n")
     if (responseModel) trace.itemValue(`model`, responseModel)
     trace.itemValue(`🏁 finish reason`, finishReason)
-    if (usage) {
+    if (usage?.total_tokens) {
         trace.itemValue(
             `🪙 tokens`,
             `${usage.total_tokens} total, ${usage.prompt_tokens} prompt, ${usage.completion_tokens} completion`

diff --git a/packages/core/src/promptdom.ts b/packages/core/src/promptdom.ts
@@ -285,7 +285,7 @@ function renderDefNode(def: PromptDefNode): string {
 
     let res: string
     if (name && fenceFormat === "xml") {
-        res = `\n<${name}${dtype ? ` lang="${dtype}"` : ""}${filename ? ` file="${filename}"` : ""}${schema ? ` schema=${schema}` : ""}${diffFormat}>\n${body}</${name}>\n`
+        res = `\n<${name}${dtype ? ` lang="${dtype}"` : ""}${filename ? ` file="${filename}"` : ""}${schema ? ` schema=${schema}` : ""}${diffFormat}>\n${body}<${name}>\n`
     } else if (fenceFormat === "none") {
         res = `\n${name ? name + ":\n" : ""}${body}\n`
     } else {

diff --git a/packages/core/src/runpromptcontext.ts b/packages/core/src/runpromptcontext.ts
@@ -33,7 +33,13 @@ import {
 } from "./parameters"
 import { consoleLogFormat, stdout } from "./logging"
 import { isGlobMatch } from "./glob"
-import { arrayify, logError, logVerbose, logWarn } from "./util"
+import {
+    arrayify,
+    deleteEmptyValues,
+    logError,
+    logVerbose,
+    logWarn,
+} from "./util"
 import { renderShellOutput } from "./chatrender"
 import { jinjaRender } from "./jinja"
 import { mustacheRender } from "./mustache"
@@ -503,7 +509,8 @@ export function createChatGenerationContext(
                         ...rest,
                     }
                 )
-                return res
+                if (res.error) throw res.error
+                return deleteEmptyValues(res)
             }
         )
     }

diff --git a/packages/core/src/types/prompt_template.d.ts b/packages/core/src/types/prompt_template.d.ts
@@ -2184,6 +2184,20 @@ interface INI {
     stringify(value: any): string
 }
 
+interface JSON5 {
+    /**
+     * Parses a JSON/YAML/XML string to an object
+     * @param text
+     */
+    parse(text: string | WorkspaceFile): any
+
+    /**
+     * Renders an object to a JSON5-LLM friendly string
+     * @param value
+     */
+    stringify(value: any): string
+}
+
 interface CSVStringifyOptions {
     delimiter?: string
     header?: boolean
@@ -2604,7 +2618,7 @@ interface McpServerConfig {
 
 type McpServersConfig = Record<string, Omit<McpServerConfig, "id" | "options">>
 
-type ZodTypeLike = { _def: any, safeParse: any, refine: any }
+type ZodTypeLike = { _def: any; safeParse: any; refine: any }
 
 interface ChatGenerationContext extends ChatTurnGenerationContext {
     defSchema(