From 8a92fd886582b7f12842494462a0b06da08d9b46 Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Sun, 22 Dec 2024 05:54:31 +0100 Subject: [PATCH] JSON5 helper (#964) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: ✨ add JSON5 parsing and stringifying utilities * handle errors in agent * fix duplicate git_diff tool --- docs/src/components/BuiltinAgents.mdx | 2 +- docs/src/components/BuiltinTools.mdx | 1 - .../content/docs/reference/scripts/system.mdx | 61 ++----------------- package.json | 1 + packages/cli/src/nodehost.ts | 3 +- packages/core/src/anthropic.ts | 2 +- packages/core/src/chat.ts | 10 ++- .../src/genaisrc/system.agent_web.genai.mjs | 2 +- .../src/genaisrc/system.annotations.genai.mjs | 3 +- .../src/genaisrc/system.diagrams.genai.mjs | 3 +- .../core/src/genaisrc/system.git.genai.mjs | 52 ---------------- packages/core/src/globals.ts | 6 ++ packages/core/src/llms.json | 6 +- packages/core/src/llms.ts | 16 ++++- packages/core/src/openai.ts | 2 +- packages/core/src/promptdom.ts | 2 +- packages/core/src/runpromptcontext.ts | 11 +++- packages/core/src/types/prompt_template.d.ts | 16 ++++- packages/core/src/types/prompt_type.d.ts | 5 ++ packages/core/src/util.ts | 24 ++++---- packages/sample/genaisrc/globals.genai.mjs | 17 ++++++ 21 files changed, 106 insertions(+), 139 deletions(-) create mode 100644 packages/sample/genaisrc/globals.genai.mjs diff --git a/docs/src/components/BuiltinAgents.mdx b/docs/src/components/BuiltinAgents.mdx index 590d5a5124..7303c90ab0 100644 --- a/docs/src/components/BuiltinAgents.mdx +++ b/docs/src/components/BuiltinAgents.mdx @@ -13,4 +13,4 @@ import { LinkCard } from '@astrojs/starlight/components'; - + diff --git a/docs/src/components/BuiltinTools.mdx b/docs/src/components/BuiltinTools.mdx index 7edf0b1683..155e4c2602 100644 --- a/docs/src/components/BuiltinTools.mdx +++ b/docs/src/components/BuiltinTools.mdx @@ -13,7 +13,6 @@ import { LinkCard } from '@astrojs/starlight/components'; - diff --git a/docs/src/content/docs/reference/scripts/system.mdx b/docs/src/content/docs/reference/scripts/system.mdx index 8590444c21..d85eac8b9e 100644 --- a/docs/src/content/docs/reference/scripts/system.mdx +++ b/docs/src/content/docs/reference/scripts/system.mdx @@ -390,7 +390,7 @@ system({ const model = env.vars.agentWebSearchModel defAgent( - "web-search", + "web", "search the web to accomplish tasks.", `Your are a helpful LLM agent that can use web search. Answer the question in QUERY.`, @@ -424,8 +424,7 @@ system({ lineNumbers: true, }) -$`## Annotation Format - +$`## Annotations Format Use the following format to report **file annotations** (same as GitHub Actions workflow). ::(notice|warning|error) file=,line=,endLine=,code=:: @@ -552,7 +551,8 @@ system({ title: "Generate diagrams" }) -$`Use mermaid syntax if you need to generate state diagrams, class inheritance diagrams, relationships.` +$`## Diagrams Format +Use mermaid syntax if you need to generate state diagrams, class inheritance diagrams, relationships.` ````` @@ -1080,7 +1080,6 @@ Tools to query a git repository. - tool `git_branch_default`: Gets the default branch using git. - tool `git_branch_current`: Gets the current branch using git. - tool `git_branch_list`: List all branches using git. -- tool `git_diff`: Computes file diffs using the git diff command. If the diff is too large, it returns the list of modified/added files. - tool `git_list_commits`: Generates a history of commits using the git log command. - tool `git_status`: Generates a status of the repository using git. - tool `git_last_tag`: Gets the last tag using git. @@ -1113,58 +1112,6 @@ defTool("git_branch_list", "List all branches using git.", {}, async () => { return await git.exec("branch") }) -defTool( - "git_diff", - "Computes file diffs using the git diff command. If the diff is too large, it returns the list of modified/added files.", - { - type: "object", - properties: { - base: { - type: "string", - description: "Base branch, ref, commit sha to compare against.", - }, - head: { - type: "string", - description: - "Head branch, ref, commit sha to compare. Use 'HEAD' to compare against the current branch.", - }, - staged: { - type: "boolean", - description: "Compare staged changes", - }, - nameOnly: { - type: "boolean", - description: "Show only file names", - }, - paths: { - type: "array", - description: "Paths to compare", - items: { - type: "string", - description: "File path or wildcard supported by git", - }, - }, - excludedPaths: { - type: "array", - description: "Paths to exclude", - items: { - type: "string", - description: "File path or wildcard supported by git", - }, - }, - }, - }, - async (args) => { - const { context, ...rest } = args - const res = await git.diff({ - llmify: true, - ...rest, - }) - return res - }, - { maxTokens: 20000 } -) - defTool( "git_list_commits", "Generates a history of commits using the git log command.", diff --git a/package.json b/package.json index f896fc556a..797fd1681d 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "setup": "git submodule update --init --recursive", "setup:az": "curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash", "setup:bicep": "az bicep upgrade", + "az:login": "az login --scope api://trapi/.default", "install:playwright": "yarn install playwright --with-deps", "install:force": "rm yarn.lock && yarn install && yarn --cwd docs install:force && yarn --cwd slides install:force && yarn gen:licenses", "compile-ext": "yarn --cwd packages/core run prompts:bundle && yarn --cwd packages/vscode run compile", diff --git a/packages/cli/src/nodehost.ts b/packages/cli/src/nodehost.ts index 7f11cb05e4..655f553891 100644 --- a/packages/cli/src/nodehost.ts +++ b/packages/cli/src/nodehost.ts @@ -131,7 +131,8 @@ export class NodeHost implements RuntimeHost { if (typeof value === "string") value = { model: value, source } const aliases = this._modelAliases[source] const c = aliases[id] || (aliases[id] = { source }) - if (value.model !== undefined) (c as any).model = value.model + if (value.model !== undefined && value.model !== id) + (c as any).model = value.model if (!isNaN(value.temperature)) (c as any).temperature = value.temperature } diff --git a/packages/core/src/anthropic.ts b/packages/core/src/anthropic.ts index 37d40769dc..84cc6dfce0 100644 --- a/packages/core/src/anthropic.ts +++ b/packages/core/src/anthropic.ts @@ -433,7 +433,7 @@ const completerFactory = ( trace.appendContent("\n\n") trace.itemValue(`🏁 finish reason`, finishReason) - if (usage) { + if (usage?.total_tokens) { trace.itemValue( `🪙 tokens`, `${usage.total_tokens} total, ${usage.prompt_tokens} prompt, ${usage.completion_tokens} completion` diff --git a/packages/core/src/chat.ts b/packages/core/src/chat.ts index 9a5fec9665..11a095797d 100644 --- a/packages/core/src/chat.ts +++ b/packages/core/src/chat.ts @@ -79,6 +79,7 @@ import { serializeLogProb, topLogprobsToMarkdown, } from "./logprob" +import { uniq } from "es-toolkit" export function toChatCompletionUserMessage( expanded: string, @@ -892,8 +893,15 @@ export async function executeChatSession( try { trace.startDetails(`🧠 llm chat`) - if (toolDefinitions?.length) + if (toolDefinitions?.length) { trace.detailsFenced(`🛠️ tools`, tools, "yaml") + const toolNames = toolDefinitions.map(({ spec }) => spec.name) + const duplicates = uniq(toolNames).filter( + (name, index) => toolNames.lastIndexOf(name) !== index + ) + if (duplicates.length) + throw new Error(`duplicate tools: ${duplicates.join(", ")}`) + } let genVars: Record while (true) { stats.turns++ diff --git a/packages/core/src/genaisrc/system.agent_web.genai.mjs b/packages/core/src/genaisrc/system.agent_web.genai.mjs index cb87e88a54..9963189b35 100644 --- a/packages/core/src/genaisrc/system.agent_web.genai.mjs +++ b/packages/core/src/genaisrc/system.agent_web.genai.mjs @@ -5,7 +5,7 @@ system({ const model = env.vars.agentWebSearchModel defAgent( - "web-search", + "web", "search the web to accomplish tasks.", `Your are a helpful LLM agent that can use web search. Answer the question in QUERY.`, diff --git a/packages/core/src/genaisrc/system.annotations.genai.mjs b/packages/core/src/genaisrc/system.annotations.genai.mjs index dce344fbc3..a734e09dec 100644 --- a/packages/core/src/genaisrc/system.annotations.genai.mjs +++ b/packages/core/src/genaisrc/system.annotations.genai.mjs @@ -5,8 +5,7 @@ system({ lineNumbers: true, }) -$`## Annotation Format - +$`## Annotations Format Use the following format to report **file annotations** (same as GitHub Actions workflow). ::(notice|warning|error) file=,line=,endLine=,code=:: diff --git a/packages/core/src/genaisrc/system.diagrams.genai.mjs b/packages/core/src/genaisrc/system.diagrams.genai.mjs index 790f0f2ea5..ee6a5421a3 100644 --- a/packages/core/src/genaisrc/system.diagrams.genai.mjs +++ b/packages/core/src/genaisrc/system.diagrams.genai.mjs @@ -2,4 +2,5 @@ system({ title: "Generate diagrams" }) -$`Use mermaid syntax if you need to generate state diagrams, class inheritance diagrams, relationships.` \ No newline at end of file +$`## Diagrams Format +Use mermaid syntax if you need to generate state diagrams, class inheritance diagrams, relationships.` \ No newline at end of file diff --git a/packages/core/src/genaisrc/system.git.genai.mjs b/packages/core/src/genaisrc/system.git.genai.mjs index b753cd36e4..1fb5fbeea0 100644 --- a/packages/core/src/genaisrc/system.git.genai.mjs +++ b/packages/core/src/genaisrc/system.git.genai.mjs @@ -25,58 +25,6 @@ defTool("git_branch_list", "List all branches using git.", {}, async () => { return await git.exec("branch") }) -defTool( - "git_diff", - "Computes file diffs using the git diff command. If the diff is too large, it returns the list of modified/added files.", - { - type: "object", - properties: { - base: { - type: "string", - description: "Base branch, ref, commit sha to compare against.", - }, - head: { - type: "string", - description: - "Head branch, ref, commit sha to compare. Use 'HEAD' to compare against the current branch.", - }, - staged: { - type: "boolean", - description: "Compare staged changes", - }, - nameOnly: { - type: "boolean", - description: "Show only file names", - }, - paths: { - type: "array", - description: "Paths to compare", - items: { - type: "string", - description: "File path or wildcard supported by git", - }, - }, - excludedPaths: { - type: "array", - description: "Paths to exclude", - items: { - type: "string", - description: "File path or wildcard supported by git", - }, - }, - }, - }, - async (args) => { - const { context, ...rest } = args - const res = await git.diff({ - llmify: true, - ...rest, - }) - return res - }, - { maxTokens: 20000 } -) - defTool( "git_list_commits", "Generates a history of commits using the git log command.", diff --git a/packages/core/src/globals.ts b/packages/core/src/globals.ts index 4b93bbaad9..cabab2dc9a 100644 --- a/packages/core/src/globals.ts +++ b/packages/core/src/globals.ts @@ -17,6 +17,7 @@ import { GitClient } from "./git" import { estimateTokens, truncateTextToTokens } from "./tokens" import { chunk, resolveTokenEncoder } from "./encoders" import { runtimeHost } from "./host" +import { JSON5Stringify, JSON5TryParse } from "./json5" /** * This file defines global utilities and installs them into the global context. @@ -86,6 +87,11 @@ export function installGlobals() { stringify: JSONLStringify, // Convert objects to JSONL string }) + glb.JSON5 = Object.freeze({ + parse: JSON5TryParse, + stringify: JSON5Stringify + }) + // Freeze AICI utilities with a generation function glb.AICI = Object.freeze({ gen: (options: AICIGenOptions) => { diff --git a/packages/core/src/llms.json b/packages/core/src/llms.json index 537459a406..46b17b071d 100644 --- a/packages/core/src/llms.json +++ b/packages/core/src/llms.json @@ -71,10 +71,10 @@ "prediction": false, "bearerToken": true, "aliases": { - "large": "gemini-2.0-flash-exp", + "large": "gemini-1.5-flash-latest", "small": "gemini-1.5-flash-latest", - "vision": "gemini-2.0-flash-exp", - "long": "gemini-2.0-flash-exp", + "vision": "gemini-1.5-flash-latest", + "long": "gemini-1.5-flash-latest", "reasoning": "gemini-2.0-flash-thinking-exp-1219", "reasoning_small": "gemini-2.0-flash-thinking-exp-1219", "embeddings": "text-embedding-004" diff --git a/packages/core/src/llms.ts b/packages/core/src/llms.ts index 62c6d8a333..d3c9743002 100644 --- a/packages/core/src/llms.ts +++ b/packages/core/src/llms.ts @@ -1,3 +1,4 @@ +import { Model } from "@anthropic-ai/sdk/resources/index.mjs" import { LARGE_MODEL_ID, SMALL_MODEL_ID, VISION_MODEL_ID } from "./constants" import { ModelConfiguration, ModelConfigurations } from "./host" import LLMS from "./llms.json" @@ -14,8 +15,19 @@ export function defaultModelConfigurations(): ModelConfigurations { ] const res = { ...(Object.fromEntries( - aliases.map((alias) => [alias, readModelAlias(alias)]) + aliases.map<[string, ModelConfiguration]>((alias) => [ + alias, + readModelAlias(alias), + ]) ) as ModelConfigurations), + ...Object.fromEntries( + Object.entries(LLMS.aliases).map<[string, ModelConfiguration]>( + ([id, model]) => [ + id, + { model, source: "default" } satisfies ModelConfiguration, + ] + ) + ), } return structuredClone(res) @@ -30,6 +42,6 @@ export function defaultModelConfigurations(): ModelConfigurations { model: candidates[0], candidates, source: "default", - }) + } satisfies ModelConfiguration) } } diff --git a/packages/core/src/openai.ts b/packages/core/src/openai.ts index 16592161f3..73b324cdc6 100644 --- a/packages/core/src/openai.ts +++ b/packages/core/src/openai.ts @@ -399,7 +399,7 @@ export const OpenAIChatCompletion: ChatCompletionHandler = async ( trace.appendContent("\n\n") if (responseModel) trace.itemValue(`model`, responseModel) trace.itemValue(`🏁 finish reason`, finishReason) - if (usage) { + if (usage?.total_tokens) { trace.itemValue( `🪙 tokens`, `${usage.total_tokens} total, ${usage.prompt_tokens} prompt, ${usage.completion_tokens} completion` diff --git a/packages/core/src/promptdom.ts b/packages/core/src/promptdom.ts index 71d9f5f63e..9d33568c99 100644 --- a/packages/core/src/promptdom.ts +++ b/packages/core/src/promptdom.ts @@ -285,7 +285,7 @@ function renderDefNode(def: PromptDefNode): string { let res: string if (name && fenceFormat === "xml") { - res = `\n<${name}${dtype ? ` lang="${dtype}"` : ""}${filename ? ` file="${filename}"` : ""}${schema ? ` schema=${schema}` : ""}${diffFormat}>\n${body}\n` + res = `\n<${name}${dtype ? ` lang="${dtype}"` : ""}${filename ? ` file="${filename}"` : ""}${schema ? ` schema=${schema}` : ""}${diffFormat}>\n${body}<${name}>\n` } else if (fenceFormat === "none") { res = `\n${name ? name + ":\n" : ""}${body}\n` } else { diff --git a/packages/core/src/runpromptcontext.ts b/packages/core/src/runpromptcontext.ts index aed17fd0f0..1a934de013 100644 --- a/packages/core/src/runpromptcontext.ts +++ b/packages/core/src/runpromptcontext.ts @@ -33,7 +33,13 @@ import { } from "./parameters" import { consoleLogFormat, stdout } from "./logging" import { isGlobMatch } from "./glob" -import { arrayify, logError, logVerbose, logWarn } from "./util" +import { + arrayify, + deleteEmptyValues, + logError, + logVerbose, + logWarn, +} from "./util" import { renderShellOutput } from "./chatrender" import { jinjaRender } from "./jinja" import { mustacheRender } from "./mustache" @@ -503,7 +509,8 @@ export function createChatGenerationContext( ...rest, } ) - return res + if (res.error) throw res.error + return deleteEmptyValues(res) } ) } diff --git a/packages/core/src/types/prompt_template.d.ts b/packages/core/src/types/prompt_template.d.ts index c4093fd092..d4f69ded6b 100644 --- a/packages/core/src/types/prompt_template.d.ts +++ b/packages/core/src/types/prompt_template.d.ts @@ -2184,6 +2184,20 @@ interface INI { stringify(value: any): string } +interface JSON5 { + /** + * Parses a JSON/YAML/XML string to an object + * @param text + */ + parse(text: string | WorkspaceFile): any + + /** + * Renders an object to a JSON5-LLM friendly string + * @param value + */ + stringify(value: any): string +} + interface CSVStringifyOptions { delimiter?: string header?: boolean @@ -2604,7 +2618,7 @@ interface McpServerConfig { type McpServersConfig = Record> -type ZodTypeLike = { _def: any, safeParse: any, refine: any } +type ZodTypeLike = { _def: any; safeParse: any; refine: any } interface ChatGenerationContext extends ChatTurnGenerationContext { defSchema( diff --git a/packages/core/src/types/prompt_type.d.ts b/packages/core/src/types/prompt_type.d.ts index 56c7042f37..f1e0e54441 100644 --- a/packages/core/src/types/prompt_type.d.ts +++ b/packages/core/src/types/prompt_type.d.ts @@ -198,6 +198,11 @@ declare var MD: MD */ declare var JSONL: JSONL +/** + * JSON5 parsing + */ +declare var JSON5: JSON5 + /** * AICI operations */ diff --git a/packages/core/src/util.ts b/packages/core/src/util.ts index a58b110737..f8e2122441 100644 --- a/packages/core/src/util.ts +++ b/packages/core/src/util.ts @@ -62,21 +62,23 @@ export function parseBoolean(s: string) { } export function deleteUndefinedValues>(o: T): T { - for (const k in o) if (o[k] === undefined) delete o[k] + if (typeof o === "object") + for (const k in o) if (o[k] === undefined) delete o[k] return o } export function deleteEmptyValues>(o: T): T { - for (const k in o) { - const v = o[k] - if ( - v === undefined || - v === null || - v === "" || - (Array.isArray(v) && !v.length) - ) - delete o[k] - } + if (typeof o === "object") + for (const k in o) { + const v = o[k] + if ( + v === undefined || + v === null || + v === "" || + (Array.isArray(v) && !v.length) + ) + delete o[k] + } return o } diff --git a/packages/sample/genaisrc/globals.genai.mjs b/packages/sample/genaisrc/globals.genai.mjs new file mode 100644 index 0000000000..74aa6128d1 --- /dev/null +++ b/packages/sample/genaisrc/globals.genai.mjs @@ -0,0 +1,17 @@ +script({ + tests: {}, + model: "small", +}) + +const data = { + name: "foo", + items: [1, 2, 3], +} + +const json5 = JSON5.parse(JSON5.stringify(data)) + +if (JSON.stringify(json5) !== JSON.stringify(data)) { + throw new Error( + "JSON5.stringify(JSON5.parse(JSON5.stringify(data))) !== data" + ) +}