From 306a5a6d7c8cc13c282d7979556c3b53172e98e3 Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Mon, 30 Sep 2024 12:19:15 -0700 Subject: [PATCH] Fix artificial error (#738) * artificial error * Add GitHub Action guide, enhance filename filtering, and add spell checker script * Fix grammar and punctuation in documentation and refine AI analysis instructions * Add file chunk size option, replace unique with uniq from es-toolkit and update dependencies * revert file chunks * Update CLI docs, improve spell checker script, and modify type definitions * Update spell checker script and documentation for clarity and changelog integration * Update script to fetch staged Markdown files for spell-checking and refine documentation * Fix syntax error by adding closing backtick in poem.genai.mts --- docs/genaisrc/genaiscript.d.ts | 8 +- .../content/docs/reference/cli/commands.md | 2 +- docs/src/content/docs/samples/cmt.mdx | 14 ++- docs/src/content/docs/samples/gai.mdx | 17 ++++ docs/src/content/docs/samples/sc.mdx | 94 +++++++++++++++++++ genaisrc/genaiscript.d.ts | 8 +- packages/auto/genaiscript.d.ts | 8 +- packages/cli/package.json | 1 + packages/cli/src/nodehost.ts | 5 +- packages/core/package.json | 1 + packages/core/src/expander.ts | 2 +- packages/core/src/fs.ts | 5 +- packages/core/src/genaisrc/genaiscript.d.ts | 8 +- .../src/genaisrc/system.changelog.genai.js | 4 +- packages/core/src/glob.ts | 5 +- packages/core/src/grep.ts | 4 +- packages/core/src/runpromptcontext.ts | 15 ++- packages/core/src/server/messages.ts | 1 + packages/core/src/systems.ts | 5 +- packages/core/src/types/prompt_template.d.ts | 6 +- packages/core/src/types/prompt_type.d.ts | 2 +- packages/core/src/util.ts | 5 - .../sample/genaisrc/blog/genaiscript.d.ts | 8 +- packages/sample/genaisrc/genaiscript.d.ts | 8 +- .../sample/genaisrc/node/genaiscript.d.ts | 8 +- packages/sample/genaisrc/poem.genai.mts | 2 +- .../sample/genaisrc/python/genaiscript.d.ts | 8 +- packages/sample/genaisrc/sc.genai.mts | 35 +++++++ .../sample/genaisrc/style/genaiscript.d.ts | 8 +- packages/sample/src/aici/genaiscript.d.ts | 8 +- packages/sample/src/errors/genaiscript.d.ts | 8 +- packages/sample/src/genaiscript.d.ts | 8 +- packages/sample/src/makecode/genaiscript.d.ts | 8 +- packages/sample/src/tla/genaiscript.d.ts | 8 +- packages/sample/src/vision/genaiscript.d.ts | 8 +- packages/vscode/genaisrc/genaiscript.d.ts | 8 +- packages/vscode/package.json | 1 + packages/vscode/src/vshost.ts | 5 +- slides/genaisrc/genaiscript.d.ts | 8 +- yarn.lock | 5 + 40 files changed, 267 insertions(+), 105 deletions(-) create mode 100644 docs/src/content/docs/samples/gai.mdx create mode 100644 docs/src/content/docs/samples/sc.mdx create mode 100644 packages/sample/genaisrc/sc.genai.mts diff --git a/docs/genaisrc/genaiscript.d.ts b/docs/genaisrc/genaiscript.d.ts index b9b339303e..5e88b248f2 100644 --- a/docs/genaisrc/genaiscript.d.ts +++ b/docs/genaisrc/genaiscript.d.ts @@ -770,12 +770,12 @@ interface DefOptions extends FenceOptions, ContextExpansionOptions, DataFilter { /** * Filename filter based on file suffix. Case insensitive. */ - endsWith?: string + endsWith?: ElementOrArray /** * Filename filter using glob syntax. */ - glob?: string + glob?: ElementOrArray /** * By default, throws an error if the value in def is empty. @@ -1908,7 +1908,7 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { options?: ChatParticipantOptions ): void defFileOutput( - pattern: string | string[], + pattern: ElementOrArray, description?: string, options?: FileOutputOptions ): void @@ -2791,7 +2791,7 @@ declare function def( * @param options expectations about the generated file content */ declare function defFileOutput( - pattern: string | string[], + pattern: ElementOrArray, description?: string, options?: FileOutputOptions ): void diff --git a/docs/src/content/docs/reference/cli/commands.md b/docs/src/content/docs/reference/cli/commands.md index 839312b527..3d778fa2b5 100644 --- a/docs/src/content/docs/reference/cli/commands.md +++ b/docs/src/content/docs/reference/cli/commands.md @@ -235,9 +235,9 @@ Options: ## `retrieval` -``` Usage: genaiscript retrieval|retreival [options] [command] +RAG support RAG support Options: diff --git a/docs/src/content/docs/samples/cmt.mdx b/docs/src/content/docs/samples/cmt.mdx index fb729ffc72..618d8236c6 100644 --- a/docs/src/content/docs/samples/cmt.mdx +++ b/docs/src/content/docs/samples/cmt.mdx @@ -9,10 +9,10 @@ import { Code } from "@astrojs/starlight/components" import source from "../../../../../packages/vscode/genaisrc/cmt.genai.mts?raw" This sample automates adding comments to source code using an LLM -and validate the changes haven't introduce any code modifications. +and validates the changes haven't introduced any code modifications. To do so, we could use a combination of tools to validate the transformer: source formatters, -compilers, linters or LLM-as-judge. +compilers, linters, or LLM-as-judge. The algorithm could be summarized as follows: @@ -45,8 +45,7 @@ if (files.length === 0) ### Processing Each File -We process each file separately, to avoid exploding the token context and keep the AI focused. -We can use [inline prompts](/genaiscript/reference/scripts/inline-prompts) to make inner queries. +We process each file separately to avoid exploding the token context and keep the AI focused. We can use [inline prompts](/genaiscript/reference/scripts/inline-prompts) to make inner queries. ```ts for (const file of files) { @@ -73,12 +72,11 @@ const res = await runPrompt( ) ``` -We provide a detailed set of instructions to the AI for how to analyze and comment on the code. +We provide a detailed set of instructions to the AI on how to analyze and comment on the code. ### Format, build, lint -At this point, we have a modified source code by an LLM. We should try to use all the available tools to validate the changes. -It is best to start with like formatters and compilers as they are deterministic and typically fast. +At this point, we have a modified source code by an LLM. We should try to use all the available tools to validate the changes. It is best to start with formatters and compilers as they are deterministic and typically fast. ### Judge results with LLM @@ -115,7 +113,7 @@ genaiscript run cmt ## Format and build -One important aspect is to normalize and valid the AI generated code. The user can provide a `format` command to run a formatter +One important aspect is to normalize and validate the AI-generated code. The user can provide a `format` command to run a formatter and a `build` command to check if the code is still valid. ```ts diff --git a/docs/src/content/docs/samples/gai.mdx b/docs/src/content/docs/samples/gai.mdx new file mode 100644 index 0000000000..f4c93a236a --- /dev/null +++ b/docs/src/content/docs/samples/gai.mdx @@ -0,0 +1,17 @@ +--- +title: GitHub Action Investigator +description: Investigate GitHub Actions failures +sidebar: + order: 100 +--- +import { Code } from "@astrojs/starlight/components" +import source from "../../../../../packages/sample/genaisrc/gai.genai.mts?raw" +import gasource from "../../../../../.github/workflows/genai-investigator.yml?raw" + +This is an in-depth guide to build a script that interactively investigates GitHub Actions failures. + +## Full source ([GitHub](https://github.com/microsoft/genaiscript/blob/main/packages/sample/genaisrc/gai.genai.mts)) + + + + diff --git a/docs/src/content/docs/samples/sc.mdx b/docs/src/content/docs/samples/sc.mdx new file mode 100644 index 0000000000..5eb10a3cd7 --- /dev/null +++ b/docs/src/content/docs/samples/sc.mdx @@ -0,0 +1,94 @@ +--- +title: Spell Checker +description: Spell check a document +sidebar: + order: 101 +--- + +import { Code } from "@astrojs/starlight/components" +import source from "../../../../../packages/sample/genaisrc/sc.genai.mts?raw" + +Automating and improving the efficiency of proofreading documents is a common need among developers and writers. This script addresses this need by checking and correcting spelling and grammar in Markdown files. + +## Code Explanation + +Starting at the top of the script, we see that it's a GenAI script, which is evident from the `.mts` extension and the `script` function call. + +```ts +script({ + title: "Spell checker", + system: ["system", "system.files", "system.diff"], + temperature: 0.1, +}) +``` + +This block sets the title of the script to "Spell checker" and specifies that it uses several system prompts, such as file operations and diff generation. The `temperature` is set to `0.1`, indicating that the script will generate output with low creativity, favoring precision. + +### Fetching Files for Checking + +Next, we check for files to process, first from the environment and then from Git if none are provided. + +```ts +let files = env.files +if (files.length === 0) { + const gitStatus = await host.exec("git diff --name-only --cached") + files = await Promise.all( + gitStatus.stdout + .split(/\r?\n/g) + .filter((filename) => /\.(md|mdx)$/.test(filename)) + .map(async (filename) => await workspace.readText(filename)) + ) +} +``` + +In this block, we're assigning files from the `env` variable, which would contain any files passed to the script. If no files are provided, we execute a Git command to get a list of all cached (staged) modified files and filter them to include only `.md` and `.mdx` files. We then read the content of these files for processing. + +### Defining the File Types to Work on + +Following this, there's a `def` call: + +```ts +def("FILES", files, { endsWith: [".md", ".mdx"] }) +``` + +This line defines `FILES` to be the array of files we gathered. The options object `{ endsWith: [".md", ".mdx"] }` tells GenAI that we're only interested in files ending with `.md` or `.mdx`. + +The `$`-prefixed backtick notation is used to write the prompt template: + +```ts +$`Fix the spelling and grammar of the content of FILES. Use diff format for small changes. + +- do NOT fix the frontmatter +- do NOT fix code regions +- do NOT fix \`code\` and \`\`\`code\`\`\` +- in .mdx files, do NOT fix inline typescript code +` +``` + +This prompt instructs GenAI to fix spelling and grammar in the content of the defined `FILES`, outputting small changes in diff format. It also specifies constraints, such as not fixing the frontmatter, code regions, inline code in markdown, and inline TypeScript code in MDX files. + +Finally, there is a `defFileOutput` call: + +```ts +defFileOutput(files, "fixed markdown or mdx files") +``` + +This call declares the intent that the script will generate "fixed markdown or mdx files" based on the input files. + +## How to Run the Script with GenAIScript CLI + +Running this spell checker script is straightforward with the GenAIScript CLI. First, ensure you have the CLI installed by following the instructions in the [GenAIScript documentation](https://microsoft.github.io/genaiscript/getting-started/installation). + +Once you have the CLI installed, navigate to your local copy of the script in your terminal or command line interface. Run the following command to execute the spell checker: + +```shell +genaiscript run sc +``` + +Remember, you do not need to specify the `.genai.mts` extension when using the `run` command. + +And there you have it—a detailed walkthrough of a GenAI spell checker script for markdown files. Happy coding and perfecting your documents! + +## Full source ([GitHub](https://github.com/microsoft/genaiscript/blob/main/packages/sample/genaisrc/sc.genai.mts)) + + diff --git a/genaisrc/genaiscript.d.ts b/genaisrc/genaiscript.d.ts index b9b339303e..5e88b248f2 100644 --- a/genaisrc/genaiscript.d.ts +++ b/genaisrc/genaiscript.d.ts @@ -770,12 +770,12 @@ interface DefOptions extends FenceOptions, ContextExpansionOptions, DataFilter { /** * Filename filter based on file suffix. Case insensitive. */ - endsWith?: string + endsWith?: ElementOrArray /** * Filename filter using glob syntax. */ - glob?: string + glob?: ElementOrArray /** * By default, throws an error if the value in def is empty. @@ -1908,7 +1908,7 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { options?: ChatParticipantOptions ): void defFileOutput( - pattern: string | string[], + pattern: ElementOrArray, description?: string, options?: FileOutputOptions ): void @@ -2791,7 +2791,7 @@ declare function def( * @param options expectations about the generated file content */ declare function defFileOutput( - pattern: string | string[], + pattern: ElementOrArray, description?: string, options?: FileOutputOptions ): void diff --git a/packages/auto/genaiscript.d.ts b/packages/auto/genaiscript.d.ts index b9b339303e..5e88b248f2 100644 --- a/packages/auto/genaiscript.d.ts +++ b/packages/auto/genaiscript.d.ts @@ -770,12 +770,12 @@ interface DefOptions extends FenceOptions, ContextExpansionOptions, DataFilter { /** * Filename filter based on file suffix. Case insensitive. */ - endsWith?: string + endsWith?: ElementOrArray /** * Filename filter using glob syntax. */ - glob?: string + glob?: ElementOrArray /** * By default, throws an error if the value in def is empty. @@ -1908,7 +1908,7 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { options?: ChatParticipantOptions ): void defFileOutput( - pattern: string | string[], + pattern: ElementOrArray, description?: string, options?: FileOutputOptions ): void @@ -2791,7 +2791,7 @@ declare function def( * @param options expectations about the generated file content */ declare function defFileOutput( - pattern: string | string[], + pattern: ElementOrArray, description?: string, options?: FileOutputOptions ): void diff --git a/packages/cli/package.json b/packages/cli/package.json index a8385a332d..fc38a053be 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -75,6 +75,7 @@ "commander": "^12.1.0", "diff": "^7.0.0", "dotenv": "^16.4.5", + "es-toolkit": "^1.22.0", "esbuild": "^0.24.0", "execa": "^9.4.0", "fs-extra": "^11.2.0", diff --git a/packages/cli/src/nodehost.ts b/packages/cli/src/nodehost.ts index d4ac6e81b6..744cf730bf 100644 --- a/packages/cli/src/nodehost.ts +++ b/packages/cli/src/nodehost.ts @@ -39,7 +39,7 @@ import { ResponseStatus, } from "../../core/src/host" import { AbortSignalOptions, TraceOptions } from "../../core/src/trace" -import { logVerbose, unique } from "../../core/src/util" +import { logVerbose } from "../../core/src/util" import { parseModelIdentifier } from "../../core/src/models" import { AuthenticationToken, @@ -51,6 +51,7 @@ import { errorMessage } from "../../core/src/error" import { BrowserManager } from "./playwright" import { shellConfirm, shellInput, shellSelect } from "./input" import { shellQuote } from "../../core/src/shell" +import { uniq } from "es-toolkit" class NodeServerManager implements ServerManager { async start(): Promise { @@ -251,7 +252,7 @@ export class NodeHost implements RuntimeHost { const gitignore = await tryReadText(".gitignore") files = await filterGitIgnore(gitignore, files) } - return unique(files) + return uniq(files) } async writeFile(name: string, content: Uint8Array): Promise { await ensureDir(dirname(name)) diff --git a/packages/core/package.json b/packages/core/package.json index 6f0c52be24..a34d392565 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -40,6 +40,7 @@ "csv-stringify": "^6.5.1", "diff": "^7.0.0", "dotenv": "^16.4.5", + "es-toolkit": "^1.22.0", "esbuild": "^0.24.0", "fast-xml-parser": "^4.5.0", "fetch-retry": "^6.0.0", diff --git a/packages/core/src/expander.ts b/packages/core/src/expander.ts index a508a6532b..15ac31ddc0 100644 --- a/packages/core/src/expander.ts +++ b/packages/core/src/expander.ts @@ -1,5 +1,5 @@ import { Project, PromptScript } from "./ast" -import { assert, normalizeFloat, normalizeInt, unique } from "./util" +import { assert, normalizeFloat, normalizeInt } from "./util" import { MarkdownTrace } from "./trace" import { errorMessage, isCancelError } from "./error" import { diff --git a/packages/core/src/fs.ts b/packages/core/src/fs.ts index 49ed57145b..ce6be0966e 100644 --- a/packages/core/src/fs.ts +++ b/packages/core/src/fs.ts @@ -1,6 +1,7 @@ import { HTTPS_REGEX } from "./constants" import { host } from "./host" -import { unique, utf8Decode, utf8Encode } from "./util" +import { utf8Decode, utf8Encode } from "./util" +import { uniq } from "es-toolkit" export async function readText(fn: string) { const curr = await host.readFile(fn) @@ -60,7 +61,7 @@ export async function expandFiles(files: string[], excludedFiles?: string[]) { files.filter((f) => !HTTPS_REGEX.test(f)), { ignore: excludedFiles?.filter((f) => !HTTPS_REGEX.test(f)) } ) - return unique([...urls, ...others]) + return uniq([...urls, ...others]) } export function filePathOrUrlToWorkspaceFile(f: string) { diff --git a/packages/core/src/genaisrc/genaiscript.d.ts b/packages/core/src/genaisrc/genaiscript.d.ts index b9b339303e..5e88b248f2 100644 --- a/packages/core/src/genaisrc/genaiscript.d.ts +++ b/packages/core/src/genaisrc/genaiscript.d.ts @@ -770,12 +770,12 @@ interface DefOptions extends FenceOptions, ContextExpansionOptions, DataFilter { /** * Filename filter based on file suffix. Case insensitive. */ - endsWith?: string + endsWith?: ElementOrArray /** * Filename filter using glob syntax. */ - glob?: string + glob?: ElementOrArray /** * By default, throws an error if the value in def is empty. @@ -1908,7 +1908,7 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { options?: ChatParticipantOptions ): void defFileOutput( - pattern: string | string[], + pattern: ElementOrArray, description?: string, options?: FileOutputOptions ): void @@ -2791,7 +2791,7 @@ declare function def( * @param options expectations about the generated file content */ declare function defFileOutput( - pattern: string | string[], + pattern: ElementOrArray, description?: string, options?: FileOutputOptions ): void diff --git a/packages/core/src/genaisrc/system.changelog.genai.js b/packages/core/src/genaisrc/system.changelog.genai.js index 56073bd804..684d22a5ea 100644 --- a/packages/core/src/genaisrc/system.changelog.genai.js +++ b/packages/core/src/genaisrc/system.changelog.genai.js @@ -17,7 +17,7 @@ index N in the above snippets, and then be prefixed with exactly the same whites the original snippets above. See also the following examples of the expected response format. CHANGELOG: -\`\`\`changelog +\`\`\`\`\`changelog ChangeLog:1@ Description: . OriginalCode@4-6: @@ -47,5 +47,5 @@ OriginalCode@23-23: [23] ChangedCode@23-23: [23] -\`\`\` +\`\`\`\`\` ` \ No newline at end of file diff --git a/packages/core/src/glob.ts b/packages/core/src/glob.ts index ae4f5dac3b..e2e90570a7 100644 --- a/packages/core/src/glob.ts +++ b/packages/core/src/glob.ts @@ -11,7 +11,10 @@ import { arrayify } from "./util" * @param patterns - A single glob pattern or an array of glob patterns to match against. * @returns A boolean indicating if the filename matches any of the patterns. */ -export function isGlobMatch(filename: string, patterns: string | string[]) { +export function isGlobMatch( + filename: string, + patterns: ElementOrArray +) { // Convert patterns to an array and check if any pattern matches the filename return arrayify(patterns).some((pattern) => { // Perform the match using minimatch with specific options diff --git a/packages/core/src/grep.ts b/packages/core/src/grep.ts index a4c13c159b..9be15ef367 100644 --- a/packages/core/src/grep.ts +++ b/packages/core/src/grep.ts @@ -1,8 +1,8 @@ import { TraceOptions } from "./trace" import { runtimeHost } from "./host" import { JSONLTryParse } from "./jsonl" -import { unique } from "./util" import { resolveFileContent } from "./file" +import { uniq } from "es-toolkit" export async function grepSearch( query: string | RegExp, @@ -32,7 +32,7 @@ export async function grepSearch( line_number: number } }[] - const files = unique( + const files = uniq( resl .filter(({ type }) => type === "match") .map(({ data }) => data.path.text) diff --git a/packages/core/src/runpromptcontext.ts b/packages/core/src/runpromptcontext.ts index 0ceeda0ea6..33592f4c1e 100644 --- a/packages/core/src/runpromptcontext.ts +++ b/packages/core/src/runpromptcontext.ts @@ -114,12 +114,17 @@ export function createChatTurnGenerationContext( (body as WorkspaceFile).filename ) { const file = body as WorkspaceFile - const { glob, endsWith } = defOptions || {} + const { glob } = defOptions || {} + const endsWith = arrayify(defOptions?.endsWith) const { filename } = file if (glob && filename) { if (!isGlobMatch(filename, glob)) return undefined } - if (endsWith && !filename.endsWith(endsWith)) return undefined + if ( + endsWith.length && + !endsWith.some((ext) => filename.endsWith(ext)) + ) + return undefined appendChild(node, createDef(name, file, doptions)) } else if ( typeof body === "object" && @@ -313,7 +318,7 @@ export function createChatGenerationContext( } const defFileOutput = ( - pattern: string, + pattern: ElementOrArray, description: string, options?: FileOutputOptions ): void => { @@ -321,7 +326,9 @@ export function createChatGenerationContext( appendChild( node, createFileOutput({ - pattern: arrayify(pattern), + pattern: arrayify(pattern).map((p) => + typeof p === "string" ? p : p.filename + ), description, options, }) diff --git a/packages/core/src/server/messages.ts b/packages/core/src/server/messages.ts index 4b1f6193e9..156e39ef97 100644 --- a/packages/core/src/server/messages.ts +++ b/packages/core/src/server/messages.ts @@ -43,6 +43,7 @@ export interface PromptScriptTestRunResponse extends ResponseStatus { } export interface PromptScriptRunOptions { + filesChunkSize: number excludedFiles: string[] excludeGitIgnore: boolean runRetry: string diff --git a/packages/core/src/systems.ts b/packages/core/src/systems.ts index 6d4850cea9..df702f4caa 100644 --- a/packages/core/src/systems.ts +++ b/packages/core/src/systems.ts @@ -1,5 +1,6 @@ +import { uniq } from "es-toolkit" import { Project } from "./ast" -import { arrayify, unique } from "./util" +import { arrayify } from "./util" // Function to resolve and return a list of systems based on the provided script and project // Analyzes script options and JavaScript source code to determine applicable systems. @@ -53,7 +54,7 @@ export function resolveSystems( // Return a unique list of non-empty systems // Filters out duplicates and empty entries using unique utility - return unique(systems.filter((s) => !!s)) + return uniq(systems.filter((s) => !!s)) } // Helper function to resolve tools in the project and return their system IDs diff --git a/packages/core/src/types/prompt_template.d.ts b/packages/core/src/types/prompt_template.d.ts index 250a11ac8a..f86b5649cb 100644 --- a/packages/core/src/types/prompt_template.d.ts +++ b/packages/core/src/types/prompt_template.d.ts @@ -737,12 +737,12 @@ interface DefOptions extends FenceOptions, ContextExpansionOptions, DataFilter { /** * Filename filter based on file suffix. Case insensitive. */ - endsWith?: string + endsWith?: ElementOrArray /** * Filename filter using glob syntax. */ - glob?: string + glob?: ElementOrArray /** * By default, throws an error if the value in def is empty. @@ -1875,7 +1875,7 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { options?: ChatParticipantOptions ): void defFileOutput( - pattern: string | string[], + pattern: ElementOrArray, description?: string, options?: FileOutputOptions ): void diff --git a/packages/core/src/types/prompt_type.d.ts b/packages/core/src/types/prompt_type.d.ts index e68f36323d..fa5cfd7e77 100644 --- a/packages/core/src/types/prompt_type.d.ts +++ b/packages/core/src/types/prompt_type.d.ts @@ -83,7 +83,7 @@ declare function def( * @param options expectations about the generated file content */ declare function defFileOutput( - pattern: string | string[], + pattern: ElementOrArray, description?: string, options?: FileOutputOptions ): void diff --git a/packages/core/src/util.ts b/packages/core/src/util.ts index c0fddf99bc..5dc1149bf3 100644 --- a/packages/core/src/util.ts +++ b/packages/core/src/util.ts @@ -1,14 +1,9 @@ -import path from "path" import { GENAISCRIPT_FOLDER, HTTPS_REGEX } from "./constants" import { isCancelError, serializeError } from "./error" import { LogLevel, host } from "./host" import { YAMLStringify } from "./yaml" import { escape as HTMLEscape_ } from "html-escaper" -export function unique(strings: string[]) { - return Array.from(new Set(strings)) -} - export function trimNewlines(s: string) { return s?.replace(/^\n*/, "").replace(/\n*$/, "") } diff --git a/packages/sample/genaisrc/blog/genaiscript.d.ts b/packages/sample/genaisrc/blog/genaiscript.d.ts index b9b339303e..5e88b248f2 100644 --- a/packages/sample/genaisrc/blog/genaiscript.d.ts +++ b/packages/sample/genaisrc/blog/genaiscript.d.ts @@ -770,12 +770,12 @@ interface DefOptions extends FenceOptions, ContextExpansionOptions, DataFilter { /** * Filename filter based on file suffix. Case insensitive. */ - endsWith?: string + endsWith?: ElementOrArray /** * Filename filter using glob syntax. */ - glob?: string + glob?: ElementOrArray /** * By default, throws an error if the value in def is empty. @@ -1908,7 +1908,7 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { options?: ChatParticipantOptions ): void defFileOutput( - pattern: string | string[], + pattern: ElementOrArray, description?: string, options?: FileOutputOptions ): void @@ -2791,7 +2791,7 @@ declare function def( * @param options expectations about the generated file content */ declare function defFileOutput( - pattern: string | string[], + pattern: ElementOrArray, description?: string, options?: FileOutputOptions ): void diff --git a/packages/sample/genaisrc/genaiscript.d.ts b/packages/sample/genaisrc/genaiscript.d.ts index b9b339303e..5e88b248f2 100644 --- a/packages/sample/genaisrc/genaiscript.d.ts +++ b/packages/sample/genaisrc/genaiscript.d.ts @@ -770,12 +770,12 @@ interface DefOptions extends FenceOptions, ContextExpansionOptions, DataFilter { /** * Filename filter based on file suffix. Case insensitive. */ - endsWith?: string + endsWith?: ElementOrArray /** * Filename filter using glob syntax. */ - glob?: string + glob?: ElementOrArray /** * By default, throws an error if the value in def is empty. @@ -1908,7 +1908,7 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { options?: ChatParticipantOptions ): void defFileOutput( - pattern: string | string[], + pattern: ElementOrArray, description?: string, options?: FileOutputOptions ): void @@ -2791,7 +2791,7 @@ declare function def( * @param options expectations about the generated file content */ declare function defFileOutput( - pattern: string | string[], + pattern: ElementOrArray, description?: string, options?: FileOutputOptions ): void diff --git a/packages/sample/genaisrc/node/genaiscript.d.ts b/packages/sample/genaisrc/node/genaiscript.d.ts index b9b339303e..5e88b248f2 100644 --- a/packages/sample/genaisrc/node/genaiscript.d.ts +++ b/packages/sample/genaisrc/node/genaiscript.d.ts @@ -770,12 +770,12 @@ interface DefOptions extends FenceOptions, ContextExpansionOptions, DataFilter { /** * Filename filter based on file suffix. Case insensitive. */ - endsWith?: string + endsWith?: ElementOrArray /** * Filename filter using glob syntax. */ - glob?: string + glob?: ElementOrArray /** * By default, throws an error if the value in def is empty. @@ -1908,7 +1908,7 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { options?: ChatParticipantOptions ): void defFileOutput( - pattern: string | string[], + pattern: ElementOrArray, description?: string, options?: FileOutputOptions ): void @@ -2791,7 +2791,7 @@ declare function def( * @param options expectations about the generated file content */ declare function defFileOutput( - pattern: string | string[], + pattern: ElementOrArray, description?: string, options?: FileOutputOptions ): void diff --git a/packages/sample/genaisrc/poem.genai.mts b/packages/sample/genaisrc/poem.genai.mts index df55f31d3a..41c9bc73d6 100644 --- a/packages/sample/genaisrc/poem.genai.mts +++ b/packages/sample/genaisrc/poem.genai.mts @@ -1 +1 @@ -$`Write a short poem.` +$`Write a short poem.` \ No newline at end of file diff --git a/packages/sample/genaisrc/python/genaiscript.d.ts b/packages/sample/genaisrc/python/genaiscript.d.ts index b9b339303e..5e88b248f2 100644 --- a/packages/sample/genaisrc/python/genaiscript.d.ts +++ b/packages/sample/genaisrc/python/genaiscript.d.ts @@ -770,12 +770,12 @@ interface DefOptions extends FenceOptions, ContextExpansionOptions, DataFilter { /** * Filename filter based on file suffix. Case insensitive. */ - endsWith?: string + endsWith?: ElementOrArray /** * Filename filter using glob syntax. */ - glob?: string + glob?: ElementOrArray /** * By default, throws an error if the value in def is empty. @@ -1908,7 +1908,7 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { options?: ChatParticipantOptions ): void defFileOutput( - pattern: string | string[], + pattern: ElementOrArray, description?: string, options?: FileOutputOptions ): void @@ -2791,7 +2791,7 @@ declare function def( * @param options expectations about the generated file content */ declare function defFileOutput( - pattern: string | string[], + pattern: ElementOrArray, description?: string, options?: FileOutputOptions ): void diff --git a/packages/sample/genaisrc/sc.genai.mts b/packages/sample/genaisrc/sc.genai.mts new file mode 100644 index 0000000000..1bc2c869ae --- /dev/null +++ b/packages/sample/genaisrc/sc.genai.mts @@ -0,0 +1,35 @@ +script({ + title: "Spell checker", + system: ["system", "system.files", "system.changelog"], + temperature: 0.2, +}) + +// Get files from environment or modified files from Git if none provided +let files = env.files +if (files.length === 0) { + // If no files are provided, read all modified files + const gitStatus = await host.exec("git diff --name-only --cached") + files = await Promise.all( + gitStatus.stdout + .split(/\r?\n/g) + .filter((filename) => /\.(md|mdx)$/.test(filename)) + .map(async (filename) => await workspace.readText(filename)) + ) +} +def("FILES", files, { endsWith: [".md", ".mdx"] }) + +$`Let's take a deep breadth and analyze the spelling and grammar of the content of FILES. +If you find a spelling or grammar mistake, fix it. Use CHANGELOG file format for small changes. +If you do not find any mistakes, do not change the content. + +- only fix major errors +- use a technical documentation tone +- minimize changes; do NOT change the meaning of the content +- if the grammar is good enough, do NOT change it +- do NOT modify the frontmatter. THIS IS IMPORTANT. +- do NOT modify code regions. THIS IS IMPORTANT. +- do NOT fix \`code\` and \`\`\`code\`\`\` sections +- in .mdx files, do NOT fix inline typescript code +` + +defFileOutput(files, "fixed markdown or mdx files") diff --git a/packages/sample/genaisrc/style/genaiscript.d.ts b/packages/sample/genaisrc/style/genaiscript.d.ts index b9b339303e..5e88b248f2 100644 --- a/packages/sample/genaisrc/style/genaiscript.d.ts +++ b/packages/sample/genaisrc/style/genaiscript.d.ts @@ -770,12 +770,12 @@ interface DefOptions extends FenceOptions, ContextExpansionOptions, DataFilter { /** * Filename filter based on file suffix. Case insensitive. */ - endsWith?: string + endsWith?: ElementOrArray /** * Filename filter using glob syntax. */ - glob?: string + glob?: ElementOrArray /** * By default, throws an error if the value in def is empty. @@ -1908,7 +1908,7 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { options?: ChatParticipantOptions ): void defFileOutput( - pattern: string | string[], + pattern: ElementOrArray, description?: string, options?: FileOutputOptions ): void @@ -2791,7 +2791,7 @@ declare function def( * @param options expectations about the generated file content */ declare function defFileOutput( - pattern: string | string[], + pattern: ElementOrArray, description?: string, options?: FileOutputOptions ): void diff --git a/packages/sample/src/aici/genaiscript.d.ts b/packages/sample/src/aici/genaiscript.d.ts index b9b339303e..5e88b248f2 100644 --- a/packages/sample/src/aici/genaiscript.d.ts +++ b/packages/sample/src/aici/genaiscript.d.ts @@ -770,12 +770,12 @@ interface DefOptions extends FenceOptions, ContextExpansionOptions, DataFilter { /** * Filename filter based on file suffix. Case insensitive. */ - endsWith?: string + endsWith?: ElementOrArray /** * Filename filter using glob syntax. */ - glob?: string + glob?: ElementOrArray /** * By default, throws an error if the value in def is empty. @@ -1908,7 +1908,7 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { options?: ChatParticipantOptions ): void defFileOutput( - pattern: string | string[], + pattern: ElementOrArray, description?: string, options?: FileOutputOptions ): void @@ -2791,7 +2791,7 @@ declare function def( * @param options expectations about the generated file content */ declare function defFileOutput( - pattern: string | string[], + pattern: ElementOrArray, description?: string, options?: FileOutputOptions ): void diff --git a/packages/sample/src/errors/genaiscript.d.ts b/packages/sample/src/errors/genaiscript.d.ts index b9b339303e..5e88b248f2 100644 --- a/packages/sample/src/errors/genaiscript.d.ts +++ b/packages/sample/src/errors/genaiscript.d.ts @@ -770,12 +770,12 @@ interface DefOptions extends FenceOptions, ContextExpansionOptions, DataFilter { /** * Filename filter based on file suffix. Case insensitive. */ - endsWith?: string + endsWith?: ElementOrArray /** * Filename filter using glob syntax. */ - glob?: string + glob?: ElementOrArray /** * By default, throws an error if the value in def is empty. @@ -1908,7 +1908,7 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { options?: ChatParticipantOptions ): void defFileOutput( - pattern: string | string[], + pattern: ElementOrArray, description?: string, options?: FileOutputOptions ): void @@ -2791,7 +2791,7 @@ declare function def( * @param options expectations about the generated file content */ declare function defFileOutput( - pattern: string | string[], + pattern: ElementOrArray, description?: string, options?: FileOutputOptions ): void diff --git a/packages/sample/src/genaiscript.d.ts b/packages/sample/src/genaiscript.d.ts index b9b339303e..5e88b248f2 100644 --- a/packages/sample/src/genaiscript.d.ts +++ b/packages/sample/src/genaiscript.d.ts @@ -770,12 +770,12 @@ interface DefOptions extends FenceOptions, ContextExpansionOptions, DataFilter { /** * Filename filter based on file suffix. Case insensitive. */ - endsWith?: string + endsWith?: ElementOrArray /** * Filename filter using glob syntax. */ - glob?: string + glob?: ElementOrArray /** * By default, throws an error if the value in def is empty. @@ -1908,7 +1908,7 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { options?: ChatParticipantOptions ): void defFileOutput( - pattern: string | string[], + pattern: ElementOrArray, description?: string, options?: FileOutputOptions ): void @@ -2791,7 +2791,7 @@ declare function def( * @param options expectations about the generated file content */ declare function defFileOutput( - pattern: string | string[], + pattern: ElementOrArray, description?: string, options?: FileOutputOptions ): void diff --git a/packages/sample/src/makecode/genaiscript.d.ts b/packages/sample/src/makecode/genaiscript.d.ts index b9b339303e..5e88b248f2 100644 --- a/packages/sample/src/makecode/genaiscript.d.ts +++ b/packages/sample/src/makecode/genaiscript.d.ts @@ -770,12 +770,12 @@ interface DefOptions extends FenceOptions, ContextExpansionOptions, DataFilter { /** * Filename filter based on file suffix. Case insensitive. */ - endsWith?: string + endsWith?: ElementOrArray /** * Filename filter using glob syntax. */ - glob?: string + glob?: ElementOrArray /** * By default, throws an error if the value in def is empty. @@ -1908,7 +1908,7 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { options?: ChatParticipantOptions ): void defFileOutput( - pattern: string | string[], + pattern: ElementOrArray, description?: string, options?: FileOutputOptions ): void @@ -2791,7 +2791,7 @@ declare function def( * @param options expectations about the generated file content */ declare function defFileOutput( - pattern: string | string[], + pattern: ElementOrArray, description?: string, options?: FileOutputOptions ): void diff --git a/packages/sample/src/tla/genaiscript.d.ts b/packages/sample/src/tla/genaiscript.d.ts index b9b339303e..5e88b248f2 100644 --- a/packages/sample/src/tla/genaiscript.d.ts +++ b/packages/sample/src/tla/genaiscript.d.ts @@ -770,12 +770,12 @@ interface DefOptions extends FenceOptions, ContextExpansionOptions, DataFilter { /** * Filename filter based on file suffix. Case insensitive. */ - endsWith?: string + endsWith?: ElementOrArray /** * Filename filter using glob syntax. */ - glob?: string + glob?: ElementOrArray /** * By default, throws an error if the value in def is empty. @@ -1908,7 +1908,7 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { options?: ChatParticipantOptions ): void defFileOutput( - pattern: string | string[], + pattern: ElementOrArray, description?: string, options?: FileOutputOptions ): void @@ -2791,7 +2791,7 @@ declare function def( * @param options expectations about the generated file content */ declare function defFileOutput( - pattern: string | string[], + pattern: ElementOrArray, description?: string, options?: FileOutputOptions ): void diff --git a/packages/sample/src/vision/genaiscript.d.ts b/packages/sample/src/vision/genaiscript.d.ts index b9b339303e..5e88b248f2 100644 --- a/packages/sample/src/vision/genaiscript.d.ts +++ b/packages/sample/src/vision/genaiscript.d.ts @@ -770,12 +770,12 @@ interface DefOptions extends FenceOptions, ContextExpansionOptions, DataFilter { /** * Filename filter based on file suffix. Case insensitive. */ - endsWith?: string + endsWith?: ElementOrArray /** * Filename filter using glob syntax. */ - glob?: string + glob?: ElementOrArray /** * By default, throws an error if the value in def is empty. @@ -1908,7 +1908,7 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { options?: ChatParticipantOptions ): void defFileOutput( - pattern: string | string[], + pattern: ElementOrArray, description?: string, options?: FileOutputOptions ): void @@ -2791,7 +2791,7 @@ declare function def( * @param options expectations about the generated file content */ declare function defFileOutput( - pattern: string | string[], + pattern: ElementOrArray, description?: string, options?: FileOutputOptions ): void diff --git a/packages/vscode/genaisrc/genaiscript.d.ts b/packages/vscode/genaisrc/genaiscript.d.ts index b9b339303e..5e88b248f2 100644 --- a/packages/vscode/genaisrc/genaiscript.d.ts +++ b/packages/vscode/genaisrc/genaiscript.d.ts @@ -770,12 +770,12 @@ interface DefOptions extends FenceOptions, ContextExpansionOptions, DataFilter { /** * Filename filter based on file suffix. Case insensitive. */ - endsWith?: string + endsWith?: ElementOrArray /** * Filename filter using glob syntax. */ - glob?: string + glob?: ElementOrArray /** * By default, throws an error if the value in def is empty. @@ -1908,7 +1908,7 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { options?: ChatParticipantOptions ): void defFileOutput( - pattern: string | string[], + pattern: ElementOrArray, description?: string, options?: FileOutputOptions ): void @@ -2791,7 +2791,7 @@ declare function def( * @param options expectations about the generated file content */ declare function defFileOutput( - pattern: string | string[], + pattern: ElementOrArray, description?: string, options?: FileOutputOptions ): void diff --git a/packages/vscode/package.json b/packages/vscode/package.json index 25f9822797..f0cc1d7ec6 100644 --- a/packages/vscode/package.json +++ b/packages/vscode/package.json @@ -406,6 +406,7 @@ "@types/vscode": "^1.93.0", "@vscode/vsce": "^3.1.0", "assert": "^2.1.0", + "es-toolkit": "^1.22.0", "eslint": "^9.11.1", "markdown-it-github-alerts": "^0.3.0", "process": "^0.11.10", diff --git a/packages/vscode/src/vshost.ts b/packages/vscode/src/vshost.ts index 4bb5510f52..f4aba35f90 100644 --- a/packages/vscode/src/vshost.ts +++ b/packages/vscode/src/vshost.ts @@ -21,8 +21,9 @@ import { Host, } from "../../core/src/host" import { TraceOptions, AbortSignalOptions } from "../../core/src/trace" -import { arrayify, unique } from "../../core/src/util" +import { arrayify } from "../../core/src/util" import { LanguageModel } from "../../core/src/chat" +import { uniq } from "es-toolkit" export class VSCodeHost extends EventTarget implements Host { dotEnvPath: string = DOT_ENV_FILENAME @@ -167,7 +168,7 @@ export class VSCodeHost extends EventTarget implements Host { const gitignore = await readFileText(this.projectUri, ".gitignore") files = await filterGitIgnore(gitignore, files) } - return unique(files) + return uniq(files) } async createDirectory(name: string): Promise { const uri = this.toProjectFileUri(name) diff --git a/slides/genaisrc/genaiscript.d.ts b/slides/genaisrc/genaiscript.d.ts index b9b339303e..5e88b248f2 100644 --- a/slides/genaisrc/genaiscript.d.ts +++ b/slides/genaisrc/genaiscript.d.ts @@ -770,12 +770,12 @@ interface DefOptions extends FenceOptions, ContextExpansionOptions, DataFilter { /** * Filename filter based on file suffix. Case insensitive. */ - endsWith?: string + endsWith?: ElementOrArray /** * Filename filter using glob syntax. */ - glob?: string + glob?: ElementOrArray /** * By default, throws an error if the value in def is empty. @@ -1908,7 +1908,7 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { options?: ChatParticipantOptions ): void defFileOutput( - pattern: string | string[], + pattern: ElementOrArray, description?: string, options?: FileOutputOptions ): void @@ -2791,7 +2791,7 @@ declare function def( * @param options expectations about the generated file content */ declare function defFileOutput( - pattern: string | string[], + pattern: ElementOrArray, description?: string, options?: FileOutputOptions ): void diff --git a/yarn.lock b/yarn.lock index 85907c4891..3b6841ed1e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2670,6 +2670,11 @@ es-errors@^1.3.0: resolved "https://registry.yarnpkg.com/es-errors/-/es-errors-1.3.0.tgz#05f75a25dab98e4fb1dcd5e1472c0546d5057c8f" integrity sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw== +es-toolkit@^1.22.0: + version "1.22.0" + resolved "https://registry.yarnpkg.com/es-toolkit/-/es-toolkit-1.22.0.tgz#7799c087cc231d1a0059dc407c25bcd1bc58d31b" + integrity sha512-0mvZ7as1Irwi+i5BNTM4Lrt5NnZGz4TcMIG8FTUuqI6GswziFGPHrmVlK+kkafSqhofKKVs6gn0GSGZiWVxAHw== + es5-ext@^0.10.35, es5-ext@^0.10.62, es5-ext@^0.10.63, es5-ext@^0.10.64, es5-ext@~0.10.14: version "0.10.64" resolved "https://registry.yarnpkg.com/es5-ext/-/es5-ext-0.10.64.tgz#12e4ffb48f1ba2ea777f1fcdd1918ef73ea21714"