From 3fa7e4bbdc8160db134b6da2ba4eb44116f44b50 Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Sat, 23 Mar 2024 11:48:31 +0000 Subject: [PATCH] a few more alt descriptions --- docs/genaisrc/genaiscript.d.ts | 199 ++++++++++++++++-- docs/src/assets/debugger.png.txt | 1 + docs/src/assets/vscode-dotenv.png.txt | 1 + .../src/assets/vscode-extensions-view.png.txt | 2 + docs/src/content/docs/index.mdx | 8 +- 5 files changed, 197 insertions(+), 14 deletions(-) create mode 100644 docs/src/assets/debugger.png.txt create mode 100644 docs/src/assets/vscode-dotenv.png.txt create mode 100644 docs/src/assets/vscode-extensions-view.png.txt diff --git a/docs/genaisrc/genaiscript.d.ts b/docs/genaisrc/genaiscript.d.ts index 16558dcfd8..dce3dae41c 100644 --- a/docs/genaisrc/genaiscript.d.ts +++ b/docs/genaisrc/genaiscript.d.ts @@ -48,7 +48,27 @@ type FileMergeHandler = ( label: string, before: string, generated: string -) => string +) => string | Promise + +interface PromptOutputProcessorResult { + /** + * Updated text + */ + text?: string + /** + * Generated files from the output + */ + files?: Record + + /** + * User defined errors + */ + annotations?: Diagnostic[] +} + +type PromptOutputProcessorHandler = ( + output: PromptGenerationOutput +) => PromptOutputProcessorResult | Promise interface UrlAdapter { contentType?: "text/plain" | "application/json" @@ -104,6 +124,15 @@ interface ModelOptions { * A deterministic integer seed to use for the model. */ seed?: number + + /** + * Default value for emitting line numbers in fenced code blocks. + */ + lineNumbers?: boolean + /** + * Use AICI controller + */ + aici?: boolean } interface PromptTemplate extends PromptLike, ModelOptions { @@ -327,11 +356,6 @@ interface ExpansionVariables { */ files: LinkedFile[] - /** - * If the contents of this variable occurs in output, an error message will be shown to the user. - */ - error: string - /** * current prompt template */ @@ -461,7 +485,13 @@ type JSONSchema = JSONSchemaObject | JSONSchemaArray interface JSONSchemaValidation { schema?: JSONSchema valid: boolean - errors?: string + error?: string +} + +interface DataFrame { + schema?: string + data: unknown + validation?: JSONSchemaValidation } interface RunPromptResult { @@ -527,6 +557,16 @@ interface Fenced { validation?: JSONSchemaValidation } +interface XMLParseOptions { + allowBooleanAttributes?: boolean + ignoreAttributes?: boolean + ignoreDeclaration?: boolean + ignorePiTags?: boolean + parseAttributeValue?: boolean + removeNSPrefix?: boolean + unpairedTags?: string[] +} + interface Parsers { /** * Parses text as a JSON5 payload @@ -590,6 +630,30 @@ interface Parsers { options?: { delimiter?: string; headers?: string[] } ): object[] | undefined + /** + * Parses a .env file + * @param content + */ + dotEnv(content: string | LinkedFile): Record + + /** + * Parses a .ini file + * @param content + */ + INI( + content: string | LinkedFile, + options?: { defaultValue?: any } + ): any | undefined + + /** + * Parses a .xml file + * @param content + */ + XML( + content: string | LinkedFile, + options?: { defaultValue?: any } & XMLParseOptions + ): any | undefined + /** * Estimates the number of tokens in the content. * @param content content to tokenize @@ -608,6 +672,60 @@ interface Parsers { annotations(content: string | LinkedFile): Diagnostic[] } +interface AICIGenOptions { + /** + * Make sure the generated text is one of the options. + */ + options?: string[] + /** + * Make sure the generated text matches given regular expression. + */ + regex?: string | RegExp + /** + * Make sure the generated text matches given yacc-like grammar. + */ + yacc?: string + /** + * Make sure the generated text is a substring of the given string. + */ + substring?: string + /** + * Used together with `substring` - treat the substring as ending the substring + * (typically '"' or similar). + */ + substringEnd?: string + /** + * Store result of the generation (as bytes) into a shared variable. + */ + storeVar?: string + /** + * Stop generation when the string is generated (the result includes the string and any following bytes (from the same token)). + */ + stopAt?: string + /** + * Stop generation when the given number of tokens have been generated. + */ + maxTokens?: number +} + +interface AICINode { + type: "aici" + name: "gen" +} + +interface AICIGenNode extends AICINode { + name: "gen" + options: AICIGenOptions +} + +interface AICI { + /** + * Generate a string that matches given constraints. + * If the tokens do not map cleanly into strings, it will contain Unicode replacement characters. + */ + gen(options: AICIGenOptions): AICIGenNode +} + interface YAML { /** * Converts an object to its YAML representation @@ -620,6 +738,20 @@ interface YAML { parse(text: string): any } +interface INI { + /** + * Parses a .ini file + * @param text + */ + parse(text: string): any + + /** + * Converts an object to.ini string + * @param value + */ + stringify(value: any): string +} + interface CSV { /** * Parses a CSV string to an array of objects @@ -667,11 +799,7 @@ interface Retreival { /** * Maximum number of embeddings to use */ - topK?: number, - /** - * Retreive summaries - */ - summary?: boolean + topK?: number /** * Minimum similarity score */ @@ -716,6 +844,33 @@ interface RunPromptContext { ): Promise } +interface PromptGenerationOutput { + /** + * LLM output. + */ + text: string + + /** + * Parsed fence sections + */ + fences: Fenced[] + + /** + * Parsed data sections + */ + frames: DataFrame[] + + /** + * A map of file updates + */ + fileEdits: Record + + /** + * Generated variables, typically from AICI.gen + */ + genVars: Record +} + interface PromptContext extends RunPromptContext { script(options: PromptArgs): void system(options: PromptSystemArgs): void @@ -727,6 +882,7 @@ interface PromptContext extends RunPromptContext { fn: ChatFunctionHandler ): void defFileMerge(fn: FileMergeHandler): void + defOutput(fn: PromptOutputProcessorHandler): void defSchema( name: string, schema: JSONSchema, @@ -754,6 +910,8 @@ interface PromptContext extends RunPromptContext { fs: FileSystem YAML: YAML CSV: CSV + INI: INI + AICI: AICI } @@ -851,6 +1009,16 @@ declare var fs: FileSystem */ declare var YAML: YAML +/** + * INI parsing and stringifying. + */ +declare var INI: INI + +/** + * AICI operations + */ +declare var AICI: AICI + /** * Fetches a given URL and returns the response. * @param url @@ -906,3 +1074,10 @@ declare function runPrompt( generator: (ctx: RunPromptContext) => void | Promise, options?: ModelOptions ): Promise + + +/** + * Registers a callback to process the LLM output + * @param fn + */ +declare function defOutput(fn: PromptOutputProcessorHandler): void diff --git a/docs/src/assets/debugger.png.txt b/docs/src/assets/debugger.png.txt new file mode 100644 index 0000000000..970845ba27 --- /dev/null +++ b/docs/src/assets/debugger.png.txt @@ -0,0 +1 @@ +A screenshot of a debugging session in a code editor with a breakpoint set on a line of code. The editor is displaying several panels including the watch variables, call stack, and a terminal output. The code is partially visible with a function definition and JSON configuration data. diff --git a/docs/src/assets/vscode-dotenv.png.txt b/docs/src/assets/vscode-dotenv.png.txt new file mode 100644 index 0000000000..17f280b330 --- /dev/null +++ b/docs/src/assets/vscode-dotenv.png.txt @@ -0,0 +1 @@ +A screenshot of a Visual Studio Code interface highlighting the .env file within the scripts directory, with the file icon indicating it is a dotenv (environment variables) file. diff --git a/docs/src/assets/vscode-extensions-view.png.txt b/docs/src/assets/vscode-extensions-view.png.txt new file mode 100644 index 0000000000..cea18d48e7 --- /dev/null +++ b/docs/src/assets/vscode-extensions-view.png.txt @@ -0,0 +1,2 @@ +Icon representing the GenAIScript view in Visual Studio Code, +located in the activity bar on the left side of the screen. diff --git a/docs/src/content/docs/index.mdx b/docs/src/content/docs/index.mdx index 7a6b602b3c..f2b86074f5 100644 --- a/docs/src/content/docs/index.mdx +++ b/docs/src/content/docs/index.mdx @@ -15,10 +15,14 @@ hero: link: https://github.com/microsoft/genaiscript/ icon: github --- - +import { Image} from "astro:assets" import { Card, CardGrid } from "@astrojs/starlight/components" import { FileTree } from "@astrojs/starlight/components" +import debuggerSrc from '../../assets/debugger.png'; +import debuggerAlt from "../../assets/debugger.png.txt?raw" + + ```js wrap title="extract-data.genai.js" // define the context def("FILE", env.files, { endsWith: ".pdf" }) @@ -75,7 +79,7 @@ $`Summarize FILE. Today is ${new Date()}.` Edit, [debug](/genaiscript/getting-started/debugging-scripts/), [run](/genaiscript/getting-started/running-scripts/) your scripts in [Visual Studio Code](/genaiscript/getting-started/installation). -![Debugging a script](../../assets/debugger.png) +{debuggerAlt}