From 0386d1ed15f872684745ebc0ba87f648d34537f7 Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Tue, 24 Sep 2024 12:20:25 +0000 Subject: [PATCH 01/11] Add comments and logging improvements, enhance cache handling, and support multiple glob patterns --- docs/genaisrc/genaiscript.d.ts | 6 +- genaisrc/genaiscript.d.ts | 6 +- packages/auto/genaiscript.d.ts | 6 +- packages/core/src/cache.ts | 29 ++- packages/core/src/diff.test.ts | 199 ++++++++++++++++++ packages/core/src/diff.ts | 7 +- packages/core/src/genaisrc/genaiscript.d.ts | 6 +- packages/core/src/glob.ts | 11 +- packages/core/src/runpromptcontext.ts | 9 +- packages/core/src/types/prompt_template.d.ts | 4 +- packages/core/src/types/prompt_type.d.ts | 2 +- .../sample/genaisrc/blog/genaiscript.d.ts | 6 +- packages/sample/genaisrc/documentor.genai.mts | 65 ++++++ packages/sample/genaisrc/genaiscript.d.ts | 6 +- .../sample/genaisrc/node/genaiscript.d.ts | 6 +- .../sample/genaisrc/python/genaiscript.d.ts | 6 +- .../sample/genaisrc/style/genaiscript.d.ts | 6 +- packages/sample/src/aici/genaiscript.d.ts | 6 +- packages/sample/src/errors/genaiscript.d.ts | 6 +- packages/sample/src/genaiscript.d.ts | 6 +- packages/sample/src/makecode/genaiscript.d.ts | 6 +- packages/sample/src/tla/genaiscript.d.ts | 6 +- packages/sample/src/vision/genaiscript.d.ts | 6 +- packages/vscode/genaisrc/genaiscript.d.ts | 6 +- slides/genaisrc/genaiscript.d.ts | 6 +- 25 files changed, 358 insertions(+), 70 deletions(-) create mode 100644 packages/sample/genaisrc/documentor.genai.mts diff --git a/docs/genaisrc/genaiscript.d.ts b/docs/genaisrc/genaiscript.d.ts index cebccd0b26..a0a2d39fdf 100644 --- a/docs/genaisrc/genaiscript.d.ts +++ b/docs/genaisrc/genaiscript.d.ts @@ -1512,7 +1512,7 @@ interface FileOutputOptions { } interface FileOutput { - pattern: string + pattern: string[] description?: string options?: FileOutputOptions } @@ -1609,7 +1609,7 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { options?: ChatParticipantOptions ): void defFileOutput( - pattern: string, + pattern: string | string[], description?: string, options?: FileOutputOptions ): void @@ -2453,7 +2453,7 @@ declare function def( * @param options expectations about the generated file content */ declare function defFileOutput( - pattern: string, + pattern: string | string[], description?: string, options?: FileOutputOptions ): void diff --git a/genaisrc/genaiscript.d.ts b/genaisrc/genaiscript.d.ts index cebccd0b26..a0a2d39fdf 100644 --- a/genaisrc/genaiscript.d.ts +++ b/genaisrc/genaiscript.d.ts @@ -1512,7 +1512,7 @@ interface FileOutputOptions { } interface FileOutput { - pattern: string + pattern: string[] description?: string options?: FileOutputOptions } @@ -1609,7 +1609,7 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { options?: ChatParticipantOptions ): void defFileOutput( - pattern: string, + pattern: string | string[], description?: string, options?: FileOutputOptions ): void @@ -2453,7 +2453,7 @@ declare function def( * @param options expectations about the generated file content */ declare function defFileOutput( - pattern: string, + pattern: string | string[], description?: string, options?: FileOutputOptions ): void diff --git a/packages/auto/genaiscript.d.ts b/packages/auto/genaiscript.d.ts index cebccd0b26..a0a2d39fdf 100644 --- a/packages/auto/genaiscript.d.ts +++ b/packages/auto/genaiscript.d.ts @@ -1512,7 +1512,7 @@ interface FileOutputOptions { } interface FileOutput { - pattern: string + pattern: string[] description?: string options?: FileOutputOptions } @@ -1609,7 +1609,7 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { options?: ChatParticipantOptions ): void defFileOutput( - pattern: string, + pattern: string | string[], description?: string, options?: FileOutputOptions ): void @@ -2453,7 +2453,7 @@ declare function def( * @param options expectations about the generated file content */ declare function defFileOutput( - pattern: string, + pattern: string | string[], description?: string, options?: FileOutputOptions ): void diff --git a/packages/core/src/cache.ts b/packages/core/src/cache.ts index 0ed84aabab..2566e544f4 100644 --- a/packages/core/src/cache.ts +++ b/packages/core/src/cache.ts @@ -5,29 +5,36 @@ import { CHANGE } from "./constants" import { TraceOptions } from "./trace" import { CORE_VERSION } from "./version" +// Represents a cache entry with a hashed identifier, key, and value export type CacheEntry = { sha: string; key: K; val: V } +// A cache class that manages entries stored in JSONL format export class JSONLineCache extends EventTarget { private _entries: Record> + // Constructor is private to enforce the use of byName factory method private constructor(public readonly name: string) { super() } + // Factory method to create or retrieve an existing cache by name static byName(name: string): JSONLineCache { - name = name.replace(/[^a-z0-9_]/gi, "_") + name = name.replace(/[^a-z0-9_]/gi, "_") // Sanitize name to valid identifier const key = "cacheKV." + name - if (host.userState[key]) return host.userState[key] + if (host.userState[key]) return host.userState[key] // Return if already exists const r = new JSONLineCache(name) host.userState[key] = r return r } + // Get the folder path for the cache storage private folder() { return dotGenaiscriptPath("cache", this.name) } + // Get the full path to the cache file private path() { return host.resolvePath(this.folder(), "db.jsonl") } + // Initialize the cache by loading entries from the file private async initialize() { if (this._entries) return this._entries = {} @@ -35,10 +42,11 @@ export class JSONLineCache extends EventTarget { const objs: CacheEntry[] = await readJSONL(this.path()) let numdup = 0 for (const obj of objs) { - if (this._entries[obj.sha]) numdup++ + if (this._entries[obj.sha]) numdup++ // Count duplicates this._entries[obj.sha] = obj } if (2 * numdup > objs.length) { + // Rewrite file if too many duplicates; preserves entry order // if too many duplicates, rewrite the file // keep the order of entries await writeJSONL( @@ -48,43 +56,50 @@ export class JSONLineCache extends EventTarget { } } + // Retrieve all keys from the cache async keys(): Promise { await this.initialize() return Object.values(this._entries).map((kv) => kv.key) } + // Retrieve all entries from the cache async entries(): Promise[]> { await this.initialize() return Object.values(this._entries).map((e) => ({ ...e })) } + // Retrieve a specific entry by its SHA identifier async getEntryBySha(sha: string) { await this.initialize() return this._entries[sha] } + // Get the value associated with a specific key async get(key: K): Promise { if (key === undefined) return undefined await this.initialize() const sha = await keySHA(key) return this._entries[sha]?.val } + // Set a key-value pair in the cache, triggering change event async set(key: K, val: V, options?: TraceOptions) { const { trace } = options || {} await this.initialize() const sha = await keySHA(key) const ent = { sha, key, val } const ex = this._entries[sha] - if (ex && JSON.stringify(ex) == JSON.stringify(ent)) return + if (ex && JSON.stringify(ex) == JSON.stringify(ent)) return // No change this._entries[sha] = ent - await appendJSONL(this.path(), [ent]) + await appendJSONL(this.path(), [ent]) // Append new entry to file trace?.item(`cache ${this.name} set`) - this.dispatchEvent(new Event(CHANGE)) + this.dispatchEvent(new Event(CHANGE)) // Notify listeners of change } + // Compute SHA for a given key async getKeySHA(key: K) { await this.initialize() const sha = await keySHA(key) return sha } } +// Compute the SHA256 hash of a key for uniqueness async function keySHA(key: any) { - if (typeof key != "string") key = JSON.stringify(key) + CORE_VERSION + if (typeof key != "string") key = JSON.stringify(key) + CORE_VERSION // Normalize key return await sha256string(key) } diff --git a/packages/core/src/diff.test.ts b/packages/core/src/diff.test.ts index 20a1b709d1..a9095f7be0 100644 --- a/packages/core/src/diff.test.ts +++ b/packages/core/src/diff.test.ts @@ -87,4 +87,203 @@ describe("diff", () => { const chunks = parseLLMDiffs(source) assert.equal(chunks.length, 18) }) + + test("start offset", () => { + const source = `[6] import { CORE_VERSION } from "./version" +[7] +[8] // Represents a cache entry with a hash (sha), key, and value +[9] export type CacheEntry = { sha: string; key: K; val: V } +[10] +[11] // JSONLineCache manages a cache stored in JSONL format +[12] export class JSONLineCache extends EventTarget { +[13] private _entries: Record> +[14] private constructor(public readonly name: string) { +[15] super() +[16] } +[17] +[18] // Creates or retrieves a cache by name +[19] static byName(name: string): JSONLineCache { +[20] name = name.replace(/[^a-z0-9_]/gi, "_") // Sanitize cache name +[21] const key = "cacheKV." + name +[22] if (host.userState[key]) return host.userState[key] +[23] const r = new JSONLineCache(name) +[24] host.userState[key] = r +[25] return r +[26] } +[27] +[28] // Returns the folder path for the cache +[29] private folder() { +[30] return dotGenaiscriptPath("cache", this.name) +[31] } +[32] // Returns the full file path for the cache data +[33] private path() { +[34] return host.resolvePath(this.folder(), "db.jsonl") +[35] } +[36] // Initializes the cache entries from the JSONL file +[37] private async initialize() { +[38] if (this._entries) return +[39] this._entries = {} +[40] await host.createDirectory(this.folder()) +[41] const objs: CacheEntry[] = await readJSONL(this.path()) +[42] let numdup = 0 +[43] for (const obj of objs) { +[44] if (this._entries[obj.sha]) numdup++ // Count duplicate entries +[45] this._entries[obj.sha] = obj +[46] } +[47] if (2 * numdup > objs.length) { +[48] // If too many duplicates, rewrite the file to remove them +[49] // Keep the order of entries +[50] await writeJSONL( +[51] this.path(), +[52] objs.filter((o) => this._entries[o.sha] === o) +[53] ) +[54] } +[55] } +[56] +[57] // Returns all keys in the cache +[58] async keys(): Promise { +[59] await this.initialize() +[60] return Object.values(this._entries).map((kv) => kv.key) +[61] } +[62] // Returns all cache entries +[63] async entries(): Promise[]> { +[64] await this.initialize() +[65] return Object.values(this._entries).map((e) => ({ ...e })) +[66] } +[67] // Retrieves an entry by its hash +[68] async getEntryBySha(sha: string) { +[69] await this.initialize() +[70] return this._entries[sha] +[71] } +[72] // Retrieves a value by its key +[73] async get(key: K): Promise { +[74] if (key === undefined) return undefined +[75] await this.initialize() +[76] const sha = await keySHA(key) +[77] return this._entries[sha]?.val +[78] } +[79] // Sets a key-value pair in the cache and appends it to the file +[80] async set(key: K, val: V, options?: TraceOptions) { +[81] const { trace } = options || {} +[82] await this.initialize() +[83] const sha = await keySHA(key) +[84] const ent = { sha, key, val } +[85] const ex = this._entries[sha] +[86] if (ex && JSON.stringify(ex) == JSON.stringify(ent)) return // No change +[87] this._entries[sha] = ent +[88] await appendJSONL(this.path(), [ent]) // Add new entry to JSONL +[89] trace?.item(\`cache \${this.name} set\`) +[90] this.dispatchEvent(new Event(CHANGE)) // Notify change +[91] } +[92] // Computes the SHA for a given key +[93] async getKeySHA(key: K) { +[94] await this.initialize() +[95] const sha = await keySHA(key) +[96] return sha +[97] } +[98] } +[99] // Computes a SHA-256 hash for a key, appending the CORE_VERSION if not a string +[100] async function keySHA(key: any) { +[101] if (typeof key != "string") key = JSON.stringify(key) + CORE_VERSION +[102] return await sha256string(key) +[103] } +` + const chunks = parseLLMDiffs(source) + console.log(chunks) + }) + + test("insert after incorrect line description", () => { + const source = `[1] import { appendJSONL, readJSONL, writeJSONL } from "./jsonl" +[2] import { host, runtimeHost } from "./host" +[3] import { dotGenaiscriptPath, sha256string } from "./util" +[4] import { CHANGE } from "./constants" +[5] import { TraceOptions } from "./trace" +[6] import { CORE_VERSION } from "./version" +[7] +[8] export type CacheEntry = { sha: string; key: K; val: V } +[9] +[10] export class JSONLineCache extends EventTarget { +[11] private _entries: Record> +[12] private constructor(public readonly name: string) { +[13] super() +[14] } +[15] +[16] static byName(name: string): JSONLineCache { +[17] name = name.replace(/[^a-z0-9_]/gi, "_") +[18] const key = "cacheKV." + name +[19] if (host.userState[key]) return host.userState[key] +[20] const r = new JSONLineCache(name) +[21] host.userState[key] = r +[22] return r +[23] } +[24] +[25] private folder() { +[26] return dotGenaiscriptPath("cache", this.name) +[27] } +[28] private path() { +[29] return host.resolvePath(this.folder(), "db.jsonl") +[30] } +[31] private async initialize() { +[32] if (this._entries) return +[33] this._entries = {} +[34] await host.createDirectory(this.folder()) +[35] const objs: CacheEntry[] = await readJSONL(this.path()) +[36] let numdup = 0 +[37] for (const obj of objs) { +[38] if (this._entries[obj.sha]) numdup++ +[39] this._entries[obj.sha] = obj +[40] } +[41] if (2 * numdup > objs.length) { +[42] // if too many duplicates, rewrite the file +[43] // keep the order of entries +[44] await writeJSONL( +[45] this.path(), +[46] objs.filter((o) => this._entries[o.sha] === o) +[47] ) +[48] } +[49] } +[50] +[51] async keys(): Promise { +[52] await this.initialize() +[53] return Object.values(this._entries).map((kv) => kv.key) +[54] } +[55] async entries(): Promise[]> { +[56] await this.initialize() +[57] return Object.values(this._entries).map((e) => ({ ...e })) +[58] } +[59] async getEntryBySha(sha: string) { +[60] await this.initialize() +[61] return this._entries[sha] +[62] } +[63] async get(key: K): Promise { +[64] if (key === undefined) return undefined +[65] await this.initialize() +[66] const sha = await keySHA(key) +[67] return this._entries[sha]?.val +[68] } +[69] async set(key: K, val: V, options?: TraceOptions) { +[70] const { trace } = options || {} +[71] await this.initialize() +[72] const sha = await keySHA(key) +[73] const ent = { sha, key, val } +[74] const ex = this._entries[sha] +[75] if (ex && JSON.stringify(ex) == JSON.stringify(ent)) return +[76] this._entries[sha] = ent +[77] await appendJSONL(this.path(), [ent]) +[78] trace?.item(\`cache \${this.name} set\`) +[79] this.dispatchEvent(new Event(CHANGE)) +[80] } +[81] async getKeySHA(key: K) { +[82] await this.initialize() +[83] const sha = await keySHA(key) +[84] return sha +[85] } +[86] } +[87] async function keySHA(key: any) { +[88] if (typeof key != "string") key = JSON.stringify(key) + CORE_VERSION +[89] return await sha256string(key) +[90] }` + const chunks = parseLLMDiffs(source) + console.log(chunks) + }) }) diff --git a/packages/core/src/diff.ts b/packages/core/src/diff.ts index f4a52f8731..a458ef6271 100644 --- a/packages/core/src/diff.ts +++ b/packages/core/src/diff.ts @@ -217,9 +217,12 @@ export function applyLLMPatch(source: string, chunks: Chunk[]): string { const line = chunk.state === "deleted" ? undefined : chunk.lines[li] const linei = chunk.lineNumbers[li] - 1 - if (isNaN(linei)) throw new DiffError("missing line number") + if (isNaN(linei)) + throw new DiffError(`diff: missing or nan line number`) if (linei < 0 || linei >= lines.length) - throw new DiffError("invalid line number") + throw new DiffError( + `diff: invalid line number ${linei} in ${lines.length}` + ) lines[linei] = line } }) diff --git a/packages/core/src/genaisrc/genaiscript.d.ts b/packages/core/src/genaisrc/genaiscript.d.ts index cebccd0b26..a0a2d39fdf 100644 --- a/packages/core/src/genaisrc/genaiscript.d.ts +++ b/packages/core/src/genaisrc/genaiscript.d.ts @@ -1512,7 +1512,7 @@ interface FileOutputOptions { } interface FileOutput { - pattern: string + pattern: string[] description?: string options?: FileOutputOptions } @@ -1609,7 +1609,7 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { options?: ChatParticipantOptions ): void defFileOutput( - pattern: string, + pattern: string | string[], description?: string, options?: FileOutputOptions ): void @@ -2453,7 +2453,7 @@ declare function def( * @param options expectations about the generated file content */ declare function defFileOutput( - pattern: string, + pattern: string | string[], description?: string, options?: FileOutputOptions ): void diff --git a/packages/core/src/glob.ts b/packages/core/src/glob.ts index 18e0eca425..763c374169 100644 --- a/packages/core/src/glob.ts +++ b/packages/core/src/glob.ts @@ -1,8 +1,11 @@ import { minimatch } from "minimatch" +import { arrayify } from "./util" -export function isGlobMatch(filename: string, pattern: string) { - const match = minimatch(filename, pattern, { - windowsPathsNoEscape: true, +export function isGlobMatch(filename: string, patterns: string | string[]) { + return arrayify(patterns).some((pattern) => { + const match = minimatch(filename, pattern, { + windowsPathsNoEscape: true, + }) + return match }) - return match } diff --git a/packages/core/src/runpromptcontext.ts b/packages/core/src/runpromptcontext.ts index 6bd8dadaae..46ad9e3501 100644 --- a/packages/core/src/runpromptcontext.ts +++ b/packages/core/src/runpromptcontext.ts @@ -17,9 +17,8 @@ import { MarkdownTrace } from "./trace" import { GenerationOptions } from "./generation" import { promptParametersSchemaToJSONSchema } from "./parameters" import { consoleLogFormat } from "./logging" -import { resolveFileDataUri } from "./file" import { isGlobMatch } from "./glob" -import { logVerbose } from "./util" +import { arrayify, logVerbose } from "./util" import { renderShellOutput } from "./chatrender" import { jinjaRender } from "./jinja" import { mustacheRender } from "./mustache" @@ -301,7 +300,11 @@ export function createChatGenerationContext( if (pattern) appendChild( node, - createFileOutput({ pattern, description, options }) + createFileOutput({ + pattern: arrayify(pattern), + description, + options, + }) ) } diff --git a/packages/core/src/types/prompt_template.d.ts b/packages/core/src/types/prompt_template.d.ts index 97b7a8af73..026536bf9b 100644 --- a/packages/core/src/types/prompt_template.d.ts +++ b/packages/core/src/types/prompt_template.d.ts @@ -1479,7 +1479,7 @@ interface FileOutputOptions { } interface FileOutput { - pattern: string + pattern: string[] description?: string options?: FileOutputOptions } @@ -1576,7 +1576,7 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { options?: ChatParticipantOptions ): void defFileOutput( - pattern: string, + pattern: string | string[], description?: string, options?: FileOutputOptions ): void diff --git a/packages/core/src/types/prompt_type.d.ts b/packages/core/src/types/prompt_type.d.ts index 59e0cf07f4..06fa7f4655 100644 --- a/packages/core/src/types/prompt_type.d.ts +++ b/packages/core/src/types/prompt_type.d.ts @@ -77,7 +77,7 @@ declare function def( * @param options expectations about the generated file content */ declare function defFileOutput( - pattern: string, + pattern: string | string[], description?: string, options?: FileOutputOptions ): void diff --git a/packages/sample/genaisrc/blog/genaiscript.d.ts b/packages/sample/genaisrc/blog/genaiscript.d.ts index cebccd0b26..a0a2d39fdf 100644 --- a/packages/sample/genaisrc/blog/genaiscript.d.ts +++ b/packages/sample/genaisrc/blog/genaiscript.d.ts @@ -1512,7 +1512,7 @@ interface FileOutputOptions { } interface FileOutput { - pattern: string + pattern: string[] description?: string options?: FileOutputOptions } @@ -1609,7 +1609,7 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { options?: ChatParticipantOptions ): void defFileOutput( - pattern: string, + pattern: string | string[], description?: string, options?: FileOutputOptions ): void @@ -2453,7 +2453,7 @@ declare function def( * @param options expectations about the generated file content */ declare function defFileOutput( - pattern: string, + pattern: string | string[], description?: string, options?: FileOutputOptions ): void diff --git a/packages/sample/genaisrc/documentor.genai.mts b/packages/sample/genaisrc/documentor.genai.mts new file mode 100644 index 0000000000..72c22feb80 --- /dev/null +++ b/packages/sample/genaisrc/documentor.genai.mts @@ -0,0 +1,65 @@ +// https://x.com/mckaywrigley/status/1838321570969981308 +import prettier from "prettier" + +script({ + system: ["system.changelog"], +}) + +const files = env.files.filter(({ filename }) => + /\.(ts|js|py|cs|java)/i.test(filename) +) +const code = def("CODE", files, { lineNumbers: true }) + +$`You are tasked with adding comments to code in ${code} to make it more understandable for AI systems or human developers. +You should analyze it and add appropriate comments as needed. + +To add comments to this code, follow these steps: + +1. Analyze the code to understand its structure and functionality. +2. Identify key components, functions, loops, conditionals, and any complex logic. +3. Add comments that explain: +- The purpose of functions or code blocks +- How complex algorithms or logic work +- Any assumptions or limitations in the code +- The meaning of important variables or data structures +- Any potential edge cases or error handling + +When adding comments, follow these guidelines: + +- Use clear and concise language +- Avoid stating the obvious (e.g., don't just restate what the code does) +- Focus on the "why" and "how" rather than just the "what" +- Use single-line comments for brief explanations +- Use multi-line comments for longer explanations or function/class descriptions +- Always place comments above the code they refer to. do NOT place comments on the same line as code. +- If comments already exist, review and update them as needed. +- Minimize changes to existing comments. + +Your output should be the original code with your added comments. Make sure to preserve the original code's formatting and structure. +Use the CHANGELOG format. + +Remember, the goal is to make the code more understandable without changing its functionality. +Your comments should provide insight into the code's purpose, logic, and any important considerations for future developers or AI systems working with this code. +` + +defFileOutput( + env.files.map(({ filename }) => filename), + "Updated code with comments." +) +defOutputProcessor(async ({ fileEdits }) => { + for (const [filepath, edit] of Object.entries(fileEdits)) { + console.log(`formatting ${filepath}`) + const options = (await prettier.resolveConfig(filepath)) ?? {} + try { + edit.after = await prettier.format(edit.after, { + ...options, + filepath, + }) + } catch (e) { + console.error( + `prettier: error formatting ${filepath}: ${e.message}` + ) + delete fileEdits[filepath] + } + } +}) diff --git a/packages/sample/genaisrc/genaiscript.d.ts b/packages/sample/genaisrc/genaiscript.d.ts index cebccd0b26..a0a2d39fdf 100644 --- a/packages/sample/genaisrc/genaiscript.d.ts +++ b/packages/sample/genaisrc/genaiscript.d.ts @@ -1512,7 +1512,7 @@ interface FileOutputOptions { } interface FileOutput { - pattern: string + pattern: string[] description?: string options?: FileOutputOptions } @@ -1609,7 +1609,7 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { options?: ChatParticipantOptions ): void defFileOutput( - pattern: string, + pattern: string | string[], description?: string, options?: FileOutputOptions ): void @@ -2453,7 +2453,7 @@ declare function def( * @param options expectations about the generated file content */ declare function defFileOutput( - pattern: string, + pattern: string | string[], description?: string, options?: FileOutputOptions ): void diff --git a/packages/sample/genaisrc/node/genaiscript.d.ts b/packages/sample/genaisrc/node/genaiscript.d.ts index cebccd0b26..a0a2d39fdf 100644 --- a/packages/sample/genaisrc/node/genaiscript.d.ts +++ b/packages/sample/genaisrc/node/genaiscript.d.ts @@ -1512,7 +1512,7 @@ interface FileOutputOptions { } interface FileOutput { - pattern: string + pattern: string[] description?: string options?: FileOutputOptions } @@ -1609,7 +1609,7 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { options?: ChatParticipantOptions ): void defFileOutput( - pattern: string, + pattern: string | string[], description?: string, options?: FileOutputOptions ): void @@ -2453,7 +2453,7 @@ declare function def( * @param options expectations about the generated file content */ declare function defFileOutput( - pattern: string, + pattern: string | string[], description?: string, options?: FileOutputOptions ): void diff --git a/packages/sample/genaisrc/python/genaiscript.d.ts b/packages/sample/genaisrc/python/genaiscript.d.ts index cebccd0b26..a0a2d39fdf 100644 --- a/packages/sample/genaisrc/python/genaiscript.d.ts +++ b/packages/sample/genaisrc/python/genaiscript.d.ts @@ -1512,7 +1512,7 @@ interface FileOutputOptions { } interface FileOutput { - pattern: string + pattern: string[] description?: string options?: FileOutputOptions } @@ -1609,7 +1609,7 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { options?: ChatParticipantOptions ): void defFileOutput( - pattern: string, + pattern: string | string[], description?: string, options?: FileOutputOptions ): void @@ -2453,7 +2453,7 @@ declare function def( * @param options expectations about the generated file content */ declare function defFileOutput( - pattern: string, + pattern: string | string[], description?: string, options?: FileOutputOptions ): void diff --git a/packages/sample/genaisrc/style/genaiscript.d.ts b/packages/sample/genaisrc/style/genaiscript.d.ts index cebccd0b26..a0a2d39fdf 100644 --- a/packages/sample/genaisrc/style/genaiscript.d.ts +++ b/packages/sample/genaisrc/style/genaiscript.d.ts @@ -1512,7 +1512,7 @@ interface FileOutputOptions { } interface FileOutput { - pattern: string + pattern: string[] description?: string options?: FileOutputOptions } @@ -1609,7 +1609,7 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { options?: ChatParticipantOptions ): void defFileOutput( - pattern: string, + pattern: string | string[], description?: string, options?: FileOutputOptions ): void @@ -2453,7 +2453,7 @@ declare function def( * @param options expectations about the generated file content */ declare function defFileOutput( - pattern: string, + pattern: string | string[], description?: string, options?: FileOutputOptions ): void diff --git a/packages/sample/src/aici/genaiscript.d.ts b/packages/sample/src/aici/genaiscript.d.ts index cebccd0b26..a0a2d39fdf 100644 --- a/packages/sample/src/aici/genaiscript.d.ts +++ b/packages/sample/src/aici/genaiscript.d.ts @@ -1512,7 +1512,7 @@ interface FileOutputOptions { } interface FileOutput { - pattern: string + pattern: string[] description?: string options?: FileOutputOptions } @@ -1609,7 +1609,7 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { options?: ChatParticipantOptions ): void defFileOutput( - pattern: string, + pattern: string | string[], description?: string, options?: FileOutputOptions ): void @@ -2453,7 +2453,7 @@ declare function def( * @param options expectations about the generated file content */ declare function defFileOutput( - pattern: string, + pattern: string | string[], description?: string, options?: FileOutputOptions ): void diff --git a/packages/sample/src/errors/genaiscript.d.ts b/packages/sample/src/errors/genaiscript.d.ts index cebccd0b26..a0a2d39fdf 100644 --- a/packages/sample/src/errors/genaiscript.d.ts +++ b/packages/sample/src/errors/genaiscript.d.ts @@ -1512,7 +1512,7 @@ interface FileOutputOptions { } interface FileOutput { - pattern: string + pattern: string[] description?: string options?: FileOutputOptions } @@ -1609,7 +1609,7 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { options?: ChatParticipantOptions ): void defFileOutput( - pattern: string, + pattern: string | string[], description?: string, options?: FileOutputOptions ): void @@ -2453,7 +2453,7 @@ declare function def( * @param options expectations about the generated file content */ declare function defFileOutput( - pattern: string, + pattern: string | string[], description?: string, options?: FileOutputOptions ): void diff --git a/packages/sample/src/genaiscript.d.ts b/packages/sample/src/genaiscript.d.ts index cebccd0b26..a0a2d39fdf 100644 --- a/packages/sample/src/genaiscript.d.ts +++ b/packages/sample/src/genaiscript.d.ts @@ -1512,7 +1512,7 @@ interface FileOutputOptions { } interface FileOutput { - pattern: string + pattern: string[] description?: string options?: FileOutputOptions } @@ -1609,7 +1609,7 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { options?: ChatParticipantOptions ): void defFileOutput( - pattern: string, + pattern: string | string[], description?: string, options?: FileOutputOptions ): void @@ -2453,7 +2453,7 @@ declare function def( * @param options expectations about the generated file content */ declare function defFileOutput( - pattern: string, + pattern: string | string[], description?: string, options?: FileOutputOptions ): void diff --git a/packages/sample/src/makecode/genaiscript.d.ts b/packages/sample/src/makecode/genaiscript.d.ts index cebccd0b26..a0a2d39fdf 100644 --- a/packages/sample/src/makecode/genaiscript.d.ts +++ b/packages/sample/src/makecode/genaiscript.d.ts @@ -1512,7 +1512,7 @@ interface FileOutputOptions { } interface FileOutput { - pattern: string + pattern: string[] description?: string options?: FileOutputOptions } @@ -1609,7 +1609,7 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { options?: ChatParticipantOptions ): void defFileOutput( - pattern: string, + pattern: string | string[], description?: string, options?: FileOutputOptions ): void @@ -2453,7 +2453,7 @@ declare function def( * @param options expectations about the generated file content */ declare function defFileOutput( - pattern: string, + pattern: string | string[], description?: string, options?: FileOutputOptions ): void diff --git a/packages/sample/src/tla/genaiscript.d.ts b/packages/sample/src/tla/genaiscript.d.ts index cebccd0b26..a0a2d39fdf 100644 --- a/packages/sample/src/tla/genaiscript.d.ts +++ b/packages/sample/src/tla/genaiscript.d.ts @@ -1512,7 +1512,7 @@ interface FileOutputOptions { } interface FileOutput { - pattern: string + pattern: string[] description?: string options?: FileOutputOptions } @@ -1609,7 +1609,7 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { options?: ChatParticipantOptions ): void defFileOutput( - pattern: string, + pattern: string | string[], description?: string, options?: FileOutputOptions ): void @@ -2453,7 +2453,7 @@ declare function def( * @param options expectations about the generated file content */ declare function defFileOutput( - pattern: string, + pattern: string | string[], description?: string, options?: FileOutputOptions ): void diff --git a/packages/sample/src/vision/genaiscript.d.ts b/packages/sample/src/vision/genaiscript.d.ts index cebccd0b26..a0a2d39fdf 100644 --- a/packages/sample/src/vision/genaiscript.d.ts +++ b/packages/sample/src/vision/genaiscript.d.ts @@ -1512,7 +1512,7 @@ interface FileOutputOptions { } interface FileOutput { - pattern: string + pattern: string[] description?: string options?: FileOutputOptions } @@ -1609,7 +1609,7 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { options?: ChatParticipantOptions ): void defFileOutput( - pattern: string, + pattern: string | string[], description?: string, options?: FileOutputOptions ): void @@ -2453,7 +2453,7 @@ declare function def( * @param options expectations about the generated file content */ declare function defFileOutput( - pattern: string, + pattern: string | string[], description?: string, options?: FileOutputOptions ): void diff --git a/packages/vscode/genaisrc/genaiscript.d.ts b/packages/vscode/genaisrc/genaiscript.d.ts index cebccd0b26..a0a2d39fdf 100644 --- a/packages/vscode/genaisrc/genaiscript.d.ts +++ b/packages/vscode/genaisrc/genaiscript.d.ts @@ -1512,7 +1512,7 @@ interface FileOutputOptions { } interface FileOutput { - pattern: string + pattern: string[] description?: string options?: FileOutputOptions } @@ -1609,7 +1609,7 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { options?: ChatParticipantOptions ): void defFileOutput( - pattern: string, + pattern: string | string[], description?: string, options?: FileOutputOptions ): void @@ -2453,7 +2453,7 @@ declare function def( * @param options expectations about the generated file content */ declare function defFileOutput( - pattern: string, + pattern: string | string[], description?: string, options?: FileOutputOptions ): void diff --git a/slides/genaisrc/genaiscript.d.ts b/slides/genaisrc/genaiscript.d.ts index cebccd0b26..a0a2d39fdf 100644 --- a/slides/genaisrc/genaiscript.d.ts +++ b/slides/genaisrc/genaiscript.d.ts @@ -1512,7 +1512,7 @@ interface FileOutputOptions { } interface FileOutput { - pattern: string + pattern: string[] description?: string options?: FileOutputOptions } @@ -1609,7 +1609,7 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { options?: ChatParticipantOptions ): void defFileOutput( - pattern: string, + pattern: string | string[], description?: string, options?: FileOutputOptions ): void @@ -2453,7 +2453,7 @@ declare function def( * @param options expectations about the generated file content */ declare function defFileOutput( - pattern: string, + pattern: string | string[], description?: string, options?: FileOutputOptions ): void From d72a2b3beaca4ab48f76dd0f122e101ba0685464 Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Tue, 24 Sep 2024 12:38:36 +0000 Subject: [PATCH 02/11] Add comments for code clarity and update script configurations --- .../content/docs/reference/scripts/system.mdx | 2 +- packages/core/src/cancellation.ts | 9 ++ packages/core/src/changelog.test.ts | 115 ++++++++++++++++++ packages/core/src/chatcache.ts | 7 ++ .../src/genaisrc/system.changelog.genai.js | 2 +- packages/core/src/promptrunner.ts | 7 +- packages/sample/genaisrc/documentor.genai.mts | 3 +- 7 files changed, 139 insertions(+), 6 deletions(-) diff --git a/docs/src/content/docs/reference/scripts/system.mdx b/docs/src/content/docs/reference/scripts/system.mdx index c237928ad6..7880f4a1d7 100644 --- a/docs/src/content/docs/reference/scripts/system.mdx +++ b/docs/src/content/docs/reference/scripts/system.mdx @@ -160,7 +160,7 @@ index N in the above snippets, and then be prefixed with exactly the same whites the original snippets above. See also the following examples of the expected response format. CHANGELOG: -\`\`\` +\`\`\`changelog ChangeLog:1@ Description: . OriginalCode@4-6: diff --git a/packages/core/src/cancellation.ts b/packages/core/src/cancellation.ts index cd7da0c673..eef9891588 100644 --- a/packages/core/src/cancellation.ts +++ b/packages/core/src/cancellation.ts @@ -11,17 +11,21 @@ import { CancelError } from "./error" export interface CancellationToken { /** * Is `true` when the token has been cancelled, `false` otherwise. + * This flag should be checked by operations to decide if they should terminate. */ isCancellationRequested: boolean } export class AbortSignalCancellationToken implements CancellationToken { + // Constructor takes an AbortSignal to track cancellation constructor(private readonly signal: AbortSignal) {} + // Accessor for checking if the cancellation has been requested get isCancellationRequested() { return this.signal.aborted } } +// Converts a CancellationToken to an AbortSignal if possible export function toSignal(token: CancellationToken) { return (token as any)?.signal } @@ -29,20 +33,25 @@ export function toSignal(token: CancellationToken) { export class AbortSignalCancellationController { readonly controller: AbortController readonly token: AbortSignalCancellationToken + + // Initializes the controller and creates a token with the associated signal constructor() { this.controller = new AbortController() this.token = new AbortSignalCancellationToken(this.controller.signal) } + // Aborts the ongoing operation with an optional reason abort(reason?: any) { this.controller.abort(reason) } } +// Checks if the operation has been cancelled and throws an error if so export function checkCancelled(token: CancellationToken) { if (token?.isCancellationRequested) throw new CancelError("user cancelled") } +// Represents optional cancellation behavior for an operation export interface CancellationOptions { cancellationToken?: CancellationToken } diff --git a/packages/core/src/changelog.test.ts b/packages/core/src/changelog.test.ts index 2a57950812..1d7d59a2a6 100644 --- a/packages/core/src/changelog.test.ts +++ b/packages/core/src/changelog.test.ts @@ -126,4 +126,119 @@ ChangedCode@85-88: assert.equal(res.length, 1) assert.equal(res[0].changes.length, 6) }) + + test("documentor", () => { + const source = `ChangeLog:1@packages/core/src/cancellation.ts +Description: Added comments to explain the purpose and functionality of the code. + +OriginalCode@3-10: +[3] /** +[4] * A cancellation token is passed to an asynchronous or long running +[5] * operation to request cancellation, like cancelling a request +[6] * for completion items because the user continued to type. +[7] * +[8] * To get an instance of a \`CancellationToken\` use a +[9] * {@link CancellationTokenSource}. +[10] */ +ChangedCode@3-10: +[3] /** +[4] * A cancellation token is passed to an asynchronous or long running +[5] * operation to request cancellation, like cancelling a request +[6] * for completion items because the user continued to type. +[7] * +[8] * To get an instance of a \`CancellationToken\` use a +[9] * {@link CancellationTokenSource}. +[10] * It helps manage and respond to user-initiated or programmatic cancellations. +[11] */ +OriginalCode@11-16: +[11] export interface CancellationToken { +[12] /** +[13] * Is \`true\` when the token has been cancelled, \`false\` otherwise. +[14] */ +[15] isCancellationRequested: boolean +[16] } +ChangedCode@11-16: +[11] export interface CancellationToken { +[12] /** +[13] * Represents whether a cancellation has been requested. +[14] * Is \`true\` when the token has been cancelled, \`false\` otherwise. +[15] */ +[16] isCancellationRequested: boolean +[17] } +OriginalCode@18-23: +[18] export class AbortSignalCancellationToken implements CancellationToken { +[19] constructor(private readonly signal: AbortSignal) {} +[20] get isCancellationRequested() { +[21] return this.signal.aborted +[22] } +[23] } +ChangedCode@18-23: +[18] export class AbortSignalCancellationToken implements CancellationToken { +[19] // Uses an AbortSignal to implement the CancellationToken interface +[20] constructor(private readonly signal: AbortSignal) {} +[21] // Getter that checks if the AbortSignal has been aborted +[22] get isCancellationRequested() { +[23] return this.signal.aborted +[24] } +[25] } +OriginalCode@25-27: +[25] export function toSignal(token: CancellationToken) { +[26] return (token as any)?.signal +[27] } +ChangedCode@25-27: +[25] export function toSignal(token: CancellationToken) { +[26] // Attempts to cast the token to any type and access a signal property +[27] return (token as any)?.signal +[28] } +OriginalCode@29-40: +[29] export class AbortSignalCancellationController { +[30] readonly controller: AbortController +[31] readonly token: AbortSignalCancellationToken +[32] constructor() { +[33] this.controller = new AbortController() +[34] this.token = new AbortSignalCancellationToken(this.controller.signal) +[35] } +[36] +[37] abort(reason?: any) { +[38] this.controller.abort(reason) +[39] } +[40] } +ChangedCode@29-40: +[29] export class AbortSignalCancellationController { +[30] // Holds an AbortController and its associated token +[31] readonly controller: AbortController +[32] readonly token: AbortSignalCancellationToken +[33] constructor() { +[34] // Initializes a new AbortController and token +[35] this.controller = new AbortController() +[36] this.token = new AbortSignalCancellationToken(this.controller.signal) +[37] } +[38] +[39] // Aborts the associated controller with an optional reason +[40] abort(reason?: any) { +[41] this.controller.abort(reason) +[42] } +[43] } +OriginalCode@42-44: +[42] export function checkCancelled(token: CancellationToken) { +[43] if (token?.isCancellationRequested) throw new CancelError("user cancelled") +[44] } +ChangedCode@42-44: +[42] export function checkCancelled(token: CancellationToken) { +[43] // Throws a CancelError if the cancellation has been requested +[44] if (token?.isCancellationRequested) throw new CancelError("user cancelled") +[45] } +OriginalCode@46-48: +[46] export interface CancellationOptions { +[47] cancellationToken?: CancellationToken +[48] } +ChangedCode@46-48: +[46] export interface CancellationOptions { +[47] // Optional CancellationToken for managing cancellation state +[48] cancellationToken?: CancellationToken +[49] }` + const res = parseChangeLogs(source) + console.log(res) + assert.equal(res[0].filename, "packages/core/src/cancellation.ts") + }) }) diff --git a/packages/core/src/chatcache.ts b/packages/core/src/chatcache.ts index 2fd2584a09..8423f5cc15 100644 --- a/packages/core/src/chatcache.ts +++ b/packages/core/src/chatcache.ts @@ -6,20 +6,27 @@ import { import { CHAT_CACHE } from "./constants" import { LanguageModelConfiguration } from "./host" +// Define the type for a cache key, which combines chat completion request +// with additional model options, excluding "token" and "source" from the language model configuration. export type ChatCompletionRequestCacheKey = CreateChatCompletionRequest & ModelOptions & Omit +// Define the type for a cache value, containing the response text +// and the reason for completion. export type ChatCompletationRequestCacheValue = { text: string finishReason: ChatCompletionResponse["finishReason"] } +// Define a JSON line cache type that maps cache keys to cache values. export type ChatCompletationRequestCache = JSONLineCache< ChatCompletionRequestCacheKey, ChatCompletationRequestCacheValue > +// Function to retrieve a chat completion cache. +// It uses a default cache name if none is provided. export function getChatCompletionCache( name?: string ): ChatCompletationRequestCache { diff --git a/packages/core/src/genaisrc/system.changelog.genai.js b/packages/core/src/genaisrc/system.changelog.genai.js index 3152c98e98..56073bd804 100644 --- a/packages/core/src/genaisrc/system.changelog.genai.js +++ b/packages/core/src/genaisrc/system.changelog.genai.js @@ -17,7 +17,7 @@ index N in the above snippets, and then be prefixed with exactly the same whites the original snippets above. See also the following examples of the expected response format. CHANGELOG: -\`\`\` +\`\`\`changelog ChangeLog:1@ Description: . OriginalCode@4-6: diff --git a/packages/core/src/promptrunner.ts b/packages/core/src/promptrunner.ts index 1c515423fa..a966574cf2 100644 --- a/packages/core/src/promptrunner.ts +++ b/packages/core/src/promptrunner.ts @@ -226,7 +226,7 @@ export async function runTemplate( for (const fence of fences.filter( ({ validation }) => validation?.valid !== false )) { - const { label: name, content: val } = fence + const { label: name, content: val, language } = fence const pm = /^((file|diff):?)\s+/i.exec(name) if (pm) { const kw = pm[1].toLowerCase() @@ -275,7 +275,10 @@ export async function runTemplate( } } } - } else if (/^changelog$/i.test(name)) { + } else if ( + /^changelog$/i.test(name) || + /^changelog/i.test(language) + ) { changelogs.push(val) const cls = parseChangeLogs(val) for (const changelog of cls) { diff --git a/packages/sample/genaisrc/documentor.genai.mts b/packages/sample/genaisrc/documentor.genai.mts index 72c22feb80..ca23027837 100644 --- a/packages/sample/genaisrc/documentor.genai.mts +++ b/packages/sample/genaisrc/documentor.genai.mts @@ -2,7 +2,7 @@ import prettier from "prettier" script({ - system: ["system.changelog"], + system: ["system", "system.files"], }) const files = env.files.filter(({ filename }) => @@ -36,7 +36,6 @@ When adding comments, follow these guidelines: - Minimize changes to existing comments. Your output should be the original code with your added comments. Make sure to preserve the original code's formatting and structure. -Use the CHANGELOG format. Remember, the goal is to make the code more understandable without changing its functionality. Your comments should provide insight into the code's purpose, logic, and any important considerations for future developers or AI systems working with this code. From 735d2f0062fab5cac982e420ece696757994b72f Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Tue, 24 Sep 2024 12:46:54 +0000 Subject: [PATCH 03/11] Improve comments and add validation for non-comment changes in chatcache and documentor files --- packages/core/src/chatcache.ts | 2 ++ packages/sample/genaisrc/documentor.genai.mts | 17 ++++++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/packages/core/src/chatcache.ts b/packages/core/src/chatcache.ts index 8423f5cc15..13b43b5992 100644 --- a/packages/core/src/chatcache.ts +++ b/packages/core/src/chatcache.ts @@ -20,6 +20,7 @@ export type ChatCompletationRequestCacheValue = { } // Define a JSON line cache type that maps cache keys to cache values. +// This cache stores chat completion requests and their associated responses. export type ChatCompletationRequestCache = JSONLineCache< ChatCompletionRequestCacheKey, ChatCompletationRequestCacheValue @@ -27,6 +28,7 @@ export type ChatCompletationRequestCache = JSONLineCache< // Function to retrieve a chat completion cache. // It uses a default cache name if none is provided. +// This function ensures consistent access to cached chat completions. export function getChatCompletionCache( name?: string ): ChatCompletationRequestCache { diff --git a/packages/sample/genaisrc/documentor.genai.mts b/packages/sample/genaisrc/documentor.genai.mts index ca23027837..e9fa56bd70 100644 --- a/packages/sample/genaisrc/documentor.genai.mts +++ b/packages/sample/genaisrc/documentor.genai.mts @@ -11,9 +11,9 @@ const files = env.files.filter(({ filename }) => const code = def("CODE", files, { lineNumbers: true }) $`You are tasked with adding comments to code in ${code} to make it more understandable for AI systems or human developers. -You should analyze it and add appropriate comments as needed. +You should analyze it, and add/update appropriate comments as needed. -To add comments to this code, follow these steps: +To add or update comments to this code, follow these steps: 1. Analyze the code to understand its structure and functionality. 2. Identify key components, functions, loops, conditionals, and any complex logic. @@ -24,7 +24,7 @@ To add comments to this code, follow these steps: - The meaning of important variables or data structures - Any potential edge cases or error handling -When adding comments, follow these guidelines: +When adding or updating comments, follow these guidelines: - Use clear and concise language - Avoid stating the obvious (e.g., don't just restate what the code does) @@ -47,6 +47,17 @@ defFileOutput( ) defOutputProcessor(async ({ fileEdits }) => { for (const [filepath, edit] of Object.entries(fileEdits)) { + console.log(`validate ${filepath}`) + const stripComments = (code: string) => + code + .split(/\r?\n/g) + .filter((l) => !l.trim().startsWith("//")) + .join("\n") + if (stripComments(edit.before) !== stripComments(edit.after)) { + console.error(`comments: error in ${filepath}: non-comment changes`) + delete fileEdits[filepath] + } + console.log(`formatting ${filepath}`) const options = (await prettier.resolveConfig(filepath)) ?? {} try { From a8d5a4ff8bdc661b91c4428b1eb3d4d718223cee Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Tue, 24 Sep 2024 13:03:26 +0000 Subject: [PATCH 04/11] Enhance TOML parsing and refactor code with improved comment handling and formatting --- packages/core/src/toml.ts | 10 +++ packages/sample/genaisrc/documentor.genai.mts | 79 ++++++++++--------- 2 files changed, 50 insertions(+), 39 deletions(-) diff --git a/packages/core/src/toml.ts b/packages/core/src/toml.ts index 64dc6426ba..dbb913f7cb 100644 --- a/packages/core/src/toml.ts +++ b/packages/core/src/toml.ts @@ -1,12 +1,22 @@ import { parse } from "toml" import { unfence } from "./fence" +// Function to safely parse TOML formatted text +// Accepts a string `text` and an optional `options` object with a `defaultValue` +// If parsing fails, it returns `defaultValue` instead of throwing an error export function TOMLTryParse(text: string, options?: { defaultValue?: any }) { try { + // Remove TOML fences from the text + // `unfence` is assumed to sanitize or format the text for parsing const cleaned = unfence(text, "toml") + + // Parse the cleaned TOML string using the `parse` function + // If parsing succeeds, return the parsed object const res = parse(cleaned) return res } catch (e) { + // If parsing throws an error, return the `defaultValue` provided in options + // This provides a fallback mechanism for error scenarios return options?.defaultValue } } diff --git a/packages/sample/genaisrc/documentor.genai.mts b/packages/sample/genaisrc/documentor.genai.mts index e9fa56bd70..d9d495c9ea 100644 --- a/packages/sample/genaisrc/documentor.genai.mts +++ b/packages/sample/genaisrc/documentor.genai.mts @@ -1,21 +1,35 @@ // https://x.com/mckaywrigley/status/1838321570969981308 import prettier from "prettier" -script({ - system: ["system", "system.files"], -}) +const files = env.files +for (const file of files) { + console.log(`processing ${file.filename}`) -const files = env.files.filter(({ filename }) => - /\.(ts|js|py|cs|java)/i.test(filename) -) -const code = def("CODE", files, { lineNumbers: true }) + // normalize input content + file.content = await prettify(file.filename, file.content) + // adding comments using genai + let newContent = await addComments(file) + // apply prettier to normalize format + newContent = await prettify(file.filename, newContent) + // saving + if (file.content !== newContent) { + console.log(`updating ${file.filename}`) + await workspace.writeText(file.filename, newContent) + } +} + +async function addComments(file: WorkspaceFile) { + const res = await runPrompt( + (ctx) => { + const code = ctx.def("CODE", file, { lineNumbers: true }) -$`You are tasked with adding comments to code in ${code} to make it more understandable for AI systems or human developers. + ctx.$`You are tasked with adding comments to code in ${code} to make it more understandable for AI systems or human developers. You should analyze it, and add/update appropriate comments as needed. To add or update comments to this code, follow these steps: 1. Analyze the code to understand its structure and functionality. +- If you are not familiar with the programming language, ignore the file. 2. Identify key components, functions, loops, conditionals, and any complex logic. 3. Add comments that explain: - The purpose of functions or code blocks @@ -40,36 +54,23 @@ Your output should be the original code with your added comments. Make sure to p Remember, the goal is to make the code more understandable without changing its functionality. Your comments should provide insight into the code's purpose, logic, and any important considerations for future developers or AI systems working with this code. ` + }, + { system: ["system", "system.files"] } + ) + const { text, fences } = res + const newContent = fences?.[0]?.content ?? text + return newContent +} -defFileOutput( - env.files.map(({ filename }) => filename), - "Updated code with comments." -) -defOutputProcessor(async ({ fileEdits }) => { - for (const [filepath, edit] of Object.entries(fileEdits)) { - console.log(`validate ${filepath}`) - const stripComments = (code: string) => - code - .split(/\r?\n/g) - .filter((l) => !l.trim().startsWith("//")) - .join("\n") - if (stripComments(edit.before) !== stripComments(edit.after)) { - console.error(`comments: error in ${filepath}: non-comment changes`) - delete fileEdits[filepath] - } - - console.log(`formatting ${filepath}`) - const options = (await prettier.resolveConfig(filepath)) ?? {} - try { - edit.after = await prettier.format(edit.after, { - ...options, - filepath, - }) - } catch (e) { - console.error( - `prettier: error formatting ${filepath}: ${e.message}` - ) - delete fileEdits[filepath] - } +async function prettify(filename: string, content: string) { + const options = (await prettier.resolveConfig(filename)) ?? {} + try { + return await prettier.format(content, { + ...options, + filepath: filename, + }) + } catch (e) { + console.error(`prettier: ${e.message}`) + return undefined } -}) +} From 6bdd6c9e2c8709054764dd6f2903a2934fa48074 Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Tue, 24 Sep 2024 13:24:55 +0000 Subject: [PATCH 05/11] Refactor systems resolution and enhance comment generator script logic --- packages/core/src/systems.ts | 26 ++++++++- .../{documentor.genai.mts => cmt.genai.mts} | 55 ++++++++++++++----- 2 files changed, 66 insertions(+), 15 deletions(-) rename packages/sample/genaisrc/{documentor.genai.mts => cmt.genai.mts} (60%) diff --git a/packages/core/src/systems.ts b/packages/core/src/systems.ts index ea753fc9ef..6d4850cea9 100644 --- a/packages/core/src/systems.ts +++ b/packages/core/src/systems.ts @@ -1,48 +1,72 @@ import { Project } from "./ast" import { arrayify, unique } from "./util" +// Function to resolve and return a list of systems based on the provided script and project +// Analyzes script options and JavaScript source code to determine applicable systems. export function resolveSystems( prj: Project, script: PromptSystemOptions & ModelOptions & { jsSource?: string } ) { const { jsSource } = script + // Initialize systems array from script.system, converting to array if necessary using arrayify utility const systems = arrayify(script.system).slice(0) + // If no system is defined in the script, determine systems based on jsSource if (script.system === undefined) { + // Check for schema definition in jsSource using regex const useSchema = /\Wdefschema\W/i.test(jsSource) + + // Default systems if no responseType is specified if (!script.responseType) { systems.push("system") systems.push("system.explanations") } - // select file expansion type + + // Determine additional systems based on content of jsSource + // Check for specific keywords in jsSource to decide which systems to add if (/\Wdiff\W/i.test(jsSource)) systems.push("system.diff") else if (/\Wchangelog\W/i.test(jsSource)) systems.push("system.changelog") else if (/\Wfile\W/i.test(jsSource)) { systems.push("system.files") + // Add file schema system if schema is used if (useSchema) systems.push("system.files_schema") } + // Add schema system if schema is used if (useSchema) systems.push("system.schema") + // Add annotation system if annotations, warnings, or errors are found if (/\W(annotation|warning|error)\W/i.test(jsSource)) systems.push("system.annotations") + // Add diagram system if diagrams or charts are found if (/\W(diagram|chart)\W/i.test(jsSource)) systems.push("system.diagrams") } + // Include tools-related systems if specified in the script if (script.tools) { systems.push("system.tools") + // Resolve and add each tool's systems based on its definition in the project arrayify(script.tools).forEach((tool) => systems.push(...resolveTools(prj, tool)) ) } + // Return a unique list of non-empty systems + // Filters out duplicates and empty entries using unique utility return unique(systems.filter((s) => !!s)) } +// Helper function to resolve tools in the project and return their system IDs +// Finds systems in the project associated with a specific tool function resolveTools(prj: Project, tool: string): string[] { + // Create regular expression to match tool definition in jsSource const toolsRx = new RegExp(`defTool\\s*\\(\\s*('|"|\`)${tool}`) + + // Filter project templates to find matching systems that define the tool const system = prj.templates.filter( (t) => t.isSystem && toolsRx.test(t.jsSource) ) + + // Return the IDs of matched systems return system.map(({ id }) => id) } diff --git a/packages/sample/genaisrc/documentor.genai.mts b/packages/sample/genaisrc/cmt.genai.mts similarity index 60% rename from packages/sample/genaisrc/documentor.genai.mts rename to packages/sample/genaisrc/cmt.genai.mts index d9d495c9ea..64e144bee1 100644 --- a/packages/sample/genaisrc/documentor.genai.mts +++ b/packages/sample/genaisrc/cmt.genai.mts @@ -1,28 +1,53 @@ -// https://x.com/mckaywrigley/status/1838321570969981308 import prettier from "prettier" -const files = env.files +script({ + title: "Source Code Comment Generator", +}) + +// Get files from environment or modified files from Git if none provided +let files = env.files +if (files.length === 0) { + // If no files are provided, read all modified files + files = await Promise.all( + (await host.exec("git status --porcelain")).stdout + .split("\n") + .filter((filename) => /^ [M|U]/.test(filename)) + .map( + async (filename) => + await workspace.readText(filename.replace(/^ [M|U] /, "")) + ) + ) +} + +// Process each file separately to avoid context explosion for (const file of files) { console.log(`processing ${file.filename}`) - - // normalize input content - file.content = await prettify(file.filename, file.content) - // adding comments using genai - let newContent = await addComments(file) - // apply prettier to normalize format - newContent = await prettify(file.filename, newContent) - // saving - if (file.content !== newContent) { - console.log(`updating ${file.filename}`) - await workspace.writeText(file.filename, newContent) + try { + // Normalize input content using prettier + file.content = await prettify(file.filename, file.content) + if (!file.content) continue + // Add comments to the code using a generative AI + let newContent = await addComments(file) + // Apply prettier again to ensure format normalization + newContent = await prettify(file.filename, newContent) + // Save modified content if different + if (file.content !== newContent) { + console.log(`updating ${file.filename}`) + await workspace.writeText(file.filename, newContent) + } + } catch (e) { + console.error(`error: ${e}`) } } +// Function to add comments to code async function addComments(file: WorkspaceFile) { const res = await runPrompt( (ctx) => { + // Define code snippet for AI context with line numbers const code = ctx.def("CODE", file, { lineNumbers: true }) + // AI prompt to add comments for better understanding ctx.$`You are tasked with adding comments to code in ${code} to make it more understandable for AI systems or human developers. You should analyze it, and add/update appropriate comments as needed. @@ -59,9 +84,11 @@ Your comments should provide insight into the code's purpose, logic, and any imp ) const { text, fences } = res const newContent = fences?.[0]?.content ?? text + if (!newContent) throw new Error("No content generated") return newContent } +// Function to prettify code using prettier async function prettify(filename: string, content: string) { const options = (await prettier.resolveConfig(filename)) ?? {} try { @@ -70,7 +97,7 @@ async function prettify(filename: string, content: string) { filepath: filename, }) } catch (e) { - console.error(`prettier: ${e.message}`) + console.log(`prettier error: ${e}`) return undefined } } From d54bdc996503b61dff1621aa9fbb75feb6bb9c7f Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Tue, 24 Sep 2024 13:41:12 +0000 Subject: [PATCH 06/11] Add comments, docstrings, and enhance code readability across multiple files --- .vscode/settings.json | 1 + packages/core/src/chatencoder.ts | 28 +++++++--- packages/core/src/chatrender.ts | 73 ++++++++++++++++++++------ packages/sample/genaisrc/cmt.genai.mts | 48 +++++++++++------ 4 files changed, 112 insertions(+), 38 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 7704b36272..bde2173f0b 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -15,6 +15,7 @@ "demux", "devcontainers", "dockerode", + "docstrings", "doptions", "Entra", "Evals", diff --git a/packages/core/src/chatencoder.ts b/packages/core/src/chatencoder.ts index 5a2f9e3fd6..10a3dacb76 100644 --- a/packages/core/src/chatencoder.ts +++ b/packages/core/src/chatencoder.ts @@ -7,14 +7,25 @@ import { import { encodeChat } from "gpt-tokenizer" import { logVerbose } from "./util" +/** + * Estimates the number of tokens in chat messages for a given model. + * Utilizes token encoding to provide an accurate count of tokens in text-based chat content. + * + * @param modelId - The identifier of the model being used. + * @param messages - An array of chat messages containing roles and content. + * @param tools - Optional array of tools used in chat completion. + * @returns The estimated number of tokens or 0 if no valid messages are found. + */ export function estimateChatTokens( modelId: string, messages: ChatCompletionMessageParam[], tools?: ChatCompletionTool[] ): number { + // Return 0 if there are no messages provided if (!messages?.length) return 0 try { - // does not support images + // Check if any message content includes image URLs + // Return undefined as images are not supported for token encoding if ( messages.find( (msg) => @@ -27,6 +38,7 @@ export function estimateChatTokens( ) return undefined + // Transform the messages into a format suitable for the token encoder const chat: { role: "user" | "system" | "assistant" content: string @@ -39,20 +51,24 @@ export function estimateChatTokens( role: role as "user" | "system" | "assistant", content: typeof content === "string" - ? content - : content + ? content // Use the string content directly + : content // Filter and join text parts if content is structured ?.filter(({ type }) => type === "text") .map( (c) => (c as ChatCompletionContentPartText).text ) - .join("\n"), + .join("\n"), // Join with newline for readability })) - .filter(({ content }) => !!content?.length) + .filter(({ content }) => !!content?.length) // Remove entries with empty content + + // Encode the chat messages and count the number of tokens const chatTokens = encodeChat(chat, "gpt-4") - return chatTokens.length | 0 + return chatTokens.length | 0 // Bitwise OR with 0 ensures integer return } catch (e) { + // Log any errors encountered during processing logVerbose(e) + // Fallback: Estimate token count based on JSON string length return (JSON.stringify(messages).length / 3) | 0 } } diff --git a/packages/core/src/chatrender.ts b/packages/core/src/chatrender.ts index 9127813b3f..7a72ece68d 100644 --- a/packages/core/src/chatrender.ts +++ b/packages/core/src/chatrender.ts @@ -9,17 +9,35 @@ import { JSON5TryParse } from "./json5" import { details, fenceMD } from "./markdown" import { YAMLStringify } from "./yaml" +/** + * Renders the output of a shell command. + * @param output - The shell output containing exit code, stdout, and stderr. + * @returns A formatted string representing the shell output. + */ export function renderShellOutput(output: ShellOutput) { + // Destructure the output object to retrieve exitCode, stdout, and stderr. const { exitCode, stdout, stderr } = output - return [ - exitCode !== 0 ? `EXIT_CODE: ${exitCode}` : undefined, - stdout ? `STDOUT:${fenceMD(stdout, "text")}` : undefined, - stderr ? `STDERR:${fenceMD(stderr, "text")}` : undefined, - ] - .filter((s) => s) - .join("\n\n") + return ( + [ + // Include exit code in the output only if it's non-zero. + exitCode !== 0 ? `EXIT_CODE: ${exitCode}` : undefined, + // Include stdout if it exists, formatted as text. + stdout ? `STDOUT:${fenceMD(stdout, "text")}` : undefined, + // Include stderr if it exists, formatted as text. + stderr ? `STDERR:${fenceMD(stderr, "text")}` : undefined, + ] + // Filter out undefined values from the array. + .filter((s) => s) + // Join the elements with two newlines for separation. + .join("\n\n") + ) } +/** + * Renders message content to a string. + * @param msg - The message which could be of various types. + * @returns A string representing the message content or undefined. + */ export function renderMessageContent( msg: | ChatCompletionAssistantMessageParam @@ -28,20 +46,33 @@ export function renderMessageContent( | ChatCompletionToolMessageParam ): string { const content = msg.content + // Return the content directly if it's a simple string. if (typeof content === "string") return content + // If the content is an array, process each element based on its type. else if (Array.isArray(content)) - return content - .map((c) => - c.type === "text" - ? c.text - : c.type === "refusal" - ? `refused: ${c.refusal}` - : `![](${c.image_url})` - ) - .join(` `) + return ( + content + .map((c) => + // Handle different types of content: text, refusal, and image. + c.type === "text" + ? c.text + : c.type === "refusal" + ? `refused: ${c.refusal}` + : `![](${c.image_url})` + ) + // Join the content array into a single string with spaces. + .join(` `) + ) + // Return undefined if the content is neither a string nor an array. return undefined } +/** + * Converts a list of chat messages to a markdown string. + * @param messages - Array of chat messages. + * @param options - Optional filtering options for different roles. + * @returns A formatted markdown string of the chat messages. + */ export function renderMessagesToMarkdown( messages: ChatCompletionMessageParam[], options?: { @@ -50,14 +81,17 @@ export function renderMessagesToMarkdown( assistant?: boolean } ) { + // Set default options for filtering message roles. const { system = undefined, user = undefined, assistant = true, } = options || {} + const res: string[] = [] messages ?.filter((msg) => { + // Filter messages based on their roles. switch (msg.role) { case "system": return system !== false @@ -133,11 +167,18 @@ export function renderMessagesToMarkdown( break } }) + // Join the result array into a single markdown string. return res.filter((s) => s !== undefined).join("\n") } +/** + * Parses and renders tool arguments into formatted YAML or JSON. + * @param args - The tool arguments as a string. + * @returns A formatted string in YAML or JSON. + */ function renderToolArguments(args: string) { const js = JSON5TryParse(args) + // Convert arguments to YAML if possible, otherwise keep as JSON. if (js) return fenceMD(YAMLStringify(js), "yaml") else return fenceMD(args, "json") } diff --git a/packages/sample/genaisrc/cmt.genai.mts b/packages/sample/genaisrc/cmt.genai.mts index 64e144bee1..be86e87fe0 100644 --- a/packages/sample/genaisrc/cmt.genai.mts +++ b/packages/sample/genaisrc/cmt.genai.mts @@ -42,13 +42,23 @@ for (const file of files) { // Function to add comments to code async function addComments(file: WorkspaceFile) { - const res = await runPrompt( - (ctx) => { - // Define code snippet for AI context with line numbers - const code = ctx.def("CODE", file, { lineNumbers: true }) + let { filename, content } = file - // AI prompt to add comments for better understanding - ctx.$`You are tasked with adding comments to code in ${code} to make it more understandable for AI systems or human developers. + // run twice as genai tend to be lazy + for (let i = 0; i < 2; i++) { + const res = await runPrompt( + (ctx) => { + // Define code snippet for AI context with line numbers + const code = ctx.def( + "CODE", + { filename, content }, + { lineNumbers: true } + ) + + // AI prompt to add comments for better understanding + ctx.$`You are an expert developer at all programming languages. + +You are tasked with adding comments to code in ${code} to make it more understandable for AI systems or human developers. You should analyze it, and add/update appropriate comments as needed. To add or update comments to this code, follow these steps: @@ -57,11 +67,12 @@ To add or update comments to this code, follow these steps: - If you are not familiar with the programming language, ignore the file. 2. Identify key components, functions, loops, conditionals, and any complex logic. 3. Add comments that explain: -- The purpose of functions or code blocks +- The purpose of functions or code blocks using the best comment format for that programming language. - How complex algorithms or logic work - Any assumptions or limitations in the code - The meaning of important variables or data structures - Any potential edge cases or error handling +- All function arguments and return value When adding or updating comments, follow these guidelines: @@ -70,22 +81,27 @@ When adding or updating comments, follow these guidelines: - Focus on the "why" and "how" rather than just the "what" - Use single-line comments for brief explanations - Use multi-line comments for longer explanations or function/class descriptions -- Always place comments above the code they refer to. do NOT place comments on the same line as code. +- Always place comments above the code they refer to. - If comments already exist, review and update them as needed. - Minimize changes to existing comments. +- For TypeScript functions and classes, use JSDoc comments. +- For Python functions and classes, use docstrings. Your output should be the original code with your added comments. Make sure to preserve the original code's formatting and structure. -Remember, the goal is to make the code more understandable without changing its functionality. +Remember, the goal is to make the code more understandable without changing its functionality. DO NOT MODIFY THE CODE ITSELF. Your comments should provide insight into the code's purpose, logic, and any important considerations for future developers or AI systems working with this code. ` - }, - { system: ["system", "system.files"] } - ) - const { text, fences } = res - const newContent = fences?.[0]?.content ?? text - if (!newContent) throw new Error("No content generated") - return newContent + }, + { system: ["system", "system.files"] } + ) + const { text, fences } = res + const nextContent = fences?.[0]?.content ?? text + if (!content) throw new Error("No content generated") + if (nextContent === content) break + content = nextContent + } + return content } // Function to prettify code using prettier From 5791433b923684770bddf672906128c72a41fa00 Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Tue, 24 Sep 2024 13:47:16 +0000 Subject: [PATCH 07/11] Remove defaultBranch parameter and dynamically resolve default branch from git configuration --- packages/vscode/genaisrc/prd.genai.mts | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/packages/vscode/genaisrc/prd.genai.mts b/packages/vscode/genaisrc/prd.genai.mts index f2386e56a2..c9f842037b 100644 --- a/packages/vscode/genaisrc/prd.genai.mts +++ b/packages/vscode/genaisrc/prd.genai.mts @@ -3,17 +3,14 @@ script({ description: "Generate a pull request description from the git diff", tools: ["fs"], temperature: 0.5, - parameters: { - defaultBranch: { - type: "string", - description: "The default branch of the repository", - default: "main", - }, - }, }) -// configuration -const defaultBranch = env.vars.defaultBranch +// resolve default branch +const defaultBranch = ( + await host.exec("git symbolic-ref refs/remotes/origin/HEAD") +).stdout + .replace("refs/remotes/origin/", "") + .trim() // context // compute diff with the default branch From 621c5a890bb790082f76cbb1651e91f506e11069 Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Tue, 24 Sep 2024 13:49:52 +0000 Subject: [PATCH 08/11] Add header comment to cmt.genai.mts file --- packages/sample/genaisrc/cmt.genai.mts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/sample/genaisrc/cmt.genai.mts b/packages/sample/genaisrc/cmt.genai.mts index be86e87fe0..c55697bced 100644 --- a/packages/sample/genaisrc/cmt.genai.mts +++ b/packages/sample/genaisrc/cmt.genai.mts @@ -1,3 +1,8 @@ +/** + * @title Source Code Comment Generator + * + * Modified from https://x.com/mckaywrigley/status/1838321570969981308 + */ import prettier from "prettier" script({ From 253d297a243c69f08a545e908f3f2c852ae678f8 Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Tue, 24 Sep 2024 14:18:15 +0000 Subject: [PATCH 09/11] Remove CLI run documentation, add VSCode commenter sample, and improve counting.py comments --- docs/src/content/docs/reference/cli/run.md | 206 ------------------ .../docs/reference/vscode/samples/cmt.mdx | 72 ++++++ packages/sample/src/counting.py | 24 +- .../{sample => vscode}/genaisrc/cmt.genai.mts | 34 +-- 4 files changed, 98 insertions(+), 238 deletions(-) delete mode 100644 docs/src/content/docs/reference/cli/run.md create mode 100644 docs/src/content/docs/reference/vscode/samples/cmt.mdx rename packages/{sample => vscode}/genaisrc/cmt.genai.mts (79%) diff --git a/docs/src/content/docs/reference/cli/run.md b/docs/src/content/docs/reference/cli/run.md deleted file mode 100644 index efd620fde6..0000000000 --- a/docs/src/content/docs/reference/cli/run.md +++ /dev/null @@ -1,206 +0,0 @@ ---- -title: Run -description: Learn how to execute genai scripts on files with streaming output to stdout, including usage of glob patterns, environment variables, and output options. -sidebar: - order: 1 -keywords: CLI tool execution, genai script running, stdout streaming, file globing, environment configuration ---- - -Runs a script on files and streams the LLM output to stdout or a folder from the workspace root. - -```bash -npx genaiscript run