From b69e02b3339c7c1e85b00ce88df6d21af288b3a2 Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Sun, 24 Nov 2024 21:57:16 -0800 Subject: [PATCH] Use CLI to list scripts (#893) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * templates to scripts * removing project class * fix: 🐛 return empty array for empty CSV input * refactor: ♻️ migrate Project to server/messages module * get scripts from cli * feat: add group filtering and schema validation 📝 * always read config when needed * fix: 🐛 return empty string if CSV is empty --- .../content/docs/reference/cli/commands.md | 13 ++- genaisrc/blog-generator.genai.mts | 1 + genaisrc/docs-referencer.genai.mts | 1 + genaisrc/docs-sample-generator.genai.mts | 1 + packages/cli/src/build.ts | 3 +- packages/cli/src/cli.ts | 8 +- packages/cli/src/info.ts | 7 +- packages/cli/src/nodehost.ts | 11 +- packages/cli/src/run.ts | 4 +- packages/cli/src/scripts.ts | 16 ++- packages/cli/src/server.ts | 15 +++ packages/cli/src/test.ts | 10 +- packages/core/src/ast.ts | 82 +++++++------- packages/core/src/config.ts | 4 +- packages/core/src/copy.ts | 1 - packages/core/src/csv.ts | 2 + packages/core/src/expander.ts | 9 +- packages/core/src/host.ts | 4 +- packages/core/src/parameters.ts | 5 +- packages/core/src/parser.ts | 25 ++--- packages/core/src/promptcontext.ts | 2 +- packages/core/src/promptrunner.ts | 2 +- packages/core/src/runpromptcontext.ts | 7 +- .../src/schemas/hostconfiguration.schema.json | 18 +++ packages/core/src/scripts.ts | 7 +- packages/core/src/server/client.ts | 9 ++ packages/core/src/server/messages.ts | 18 +++ packages/core/src/systems.ts | 6 +- packages/core/src/template.ts | 104 +++++++++--------- packages/core/src/testhost.ts | 7 +- packages/core/src/util.ts | 10 +- packages/sample/genaiscript.config.json | 4 + packages/vscode/src/chatparticipant.ts | 2 +- packages/vscode/src/fragmentcommands.ts | 6 +- packages/vscode/src/promptcommands.ts | 4 +- packages/vscode/src/prompttree.ts | 23 ++-- packages/vscode/src/state.ts | 23 +--- packages/vscode/src/taskprovider.ts | 2 +- packages/vscode/src/testcontroller.ts | 6 +- 39 files changed, 281 insertions(+), 201 deletions(-) create mode 100644 packages/core/src/schemas/hostconfiguration.schema.json create mode 100644 packages/sample/genaiscript.config.json diff --git a/docs/src/content/docs/reference/cli/commands.md b/docs/src/content/docs/reference/cli/commands.md index 493f45972a..fd49caa9c9 100644 --- a/docs/src/content/docs/reference/cli/commands.md +++ b/docs/src/content/docs/reference/cli/commands.md @@ -98,7 +98,7 @@ Options: -v, --verbose verbose output -pv, --promptfoo-version [version] promptfoo version, default is 0.97.0 -os, --out-summary append output summary in file - --groups groups to include or exclude. Use :! + -g, --groups groups to include or exclude. Use :! prefix to exclude -h, --help display help for command ``` @@ -111,8 +111,9 @@ Usage: genaiscript test list [options] List available tests in workspace Options: - --groups groups to include or exclude. Use :! prefix to exclude - -h, --help display help for command + -g, --groups groups to include or exclude. Use :! prefix to + exclude + -h, --help display help for command ``` ### `test view` @@ -137,7 +138,7 @@ Options: -h, --help display help for command Commands: - list List all available scripts in workspace + list [options] List all available scripts in workspace create Create a new script fix fix all definition files compile [folders...] Compile all scripts in workspace @@ -153,7 +154,9 @@ Usage: genaiscript scripts list [options] List all available scripts in workspace Options: - -h, --help display help for command + -g, --groups groups to include or exclude. Use :! prefix to + exclude + -h, --help display help for command ``` ### `scripts create` diff --git a/genaisrc/blog-generator.genai.mts b/genaisrc/blog-generator.genai.mts index c0b8c9f336..bbad4b80fe 100644 --- a/genaisrc/blog-generator.genai.mts +++ b/genaisrc/blog-generator.genai.mts @@ -13,6 +13,7 @@ script({ description: "The topic and goal of the article", }, }, + group: "docs" }) let { topic, theme } = env.vars diff --git a/genaisrc/docs-referencer.genai.mts b/genaisrc/docs-referencer.genai.mts index c2c945753d..675cb85f3e 100644 --- a/genaisrc/docs-referencer.genai.mts +++ b/genaisrc/docs-referencer.genai.mts @@ -9,6 +9,7 @@ script({ }, system: ["system", "system.files"], tools: ["fs", "md"], + group: "docs" }) const api = env.vars.api || "git" diff --git a/genaisrc/docs-sample-generator.genai.mts b/genaisrc/docs-sample-generator.genai.mts index 5da2cd6b75..e11abb18e9 100644 --- a/genaisrc/docs-sample-generator.genai.mts +++ b/genaisrc/docs-sample-generator.genai.mts @@ -9,6 +9,7 @@ script({ default: "defTool", }, }, + group: "docs" }) const api = env.vars.api + "" diff --git a/packages/cli/src/build.ts b/packages/cli/src/build.ts index 1d64c25734..9c14dc3cae 100644 --- a/packages/cli/src/build.ts +++ b/packages/cli/src/build.ts @@ -23,7 +23,8 @@ export async function buildProject(options?: { } else { let tps = arrayify(toolsPath) if (!tps?.length) { - tps = [GENAI_ANYJS_GLOB, ...arrayify(runtimeHost.config.include)] + const config = await runtimeHost.readConfig() + tps = [GENAI_ANYJS_GLOB, ...arrayify(config.include)] } tps = arrayify(tps) scriptFiles = [] diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index 914e4fec06..b50334176d 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -218,7 +218,7 @@ export async function cli() { ) .option("-os, --out-summary ", "append output summary in file") .option( - "--groups ", + "-g, --groups ", "groups to include or exclude. Use :! prefix to exclude" ) .action(scriptsTest) // Action to run the tests @@ -228,7 +228,7 @@ export async function cli() { .description("List available tests in workspace") .action(scriptTestList) // Action to list the tests .option( - "--groups ", + "-g, --groups ", "groups to include or exclude. Use :! prefix to exclude" ) @@ -245,6 +245,10 @@ export async function cli() { scripts .command("list", { isDefault: true }) .description("List all available scripts in workspace") + .option( + "-g, --groups ", + "groups to include or exclude. Use :! prefix to exclude" + ) .action(listScripts) // Action to list scripts scripts .command("create") diff --git a/packages/cli/src/info.ts b/packages/cli/src/info.ts index c9802ea9be..66fb9677ca 100644 --- a/packages/cli/src/info.ts +++ b/packages/cli/src/info.ts @@ -37,8 +37,9 @@ export async function envInfo( options?: { token?: boolean; error?: boolean } ) { const { token, error } = options || {} - const res: any = {} - res[".env"] = runtimeHost.config.envFile ?? "" + const config = await runtimeHost.readConfig() + const res: any = {} + res[".env"] = config.envFile ?? "" res.providers = [] const env = process.env @@ -103,7 +104,7 @@ async function resolveScriptsConnectionInfo( */ export async function modelInfo(script: string, options?: { token?: boolean }) { const prj = await buildProject() - const templates = prj.templates.filter( + const templates = prj.scripts.filter( (t) => !script || t.id === script || diff --git a/packages/cli/src/nodehost.ts b/packages/cli/src/nodehost.ts index 3dd1bb428a..c0a235cb99 100644 --- a/packages/cli/src/nodehost.ts +++ b/packages/cli/src/nodehost.ts @@ -59,7 +59,7 @@ import { shellConfirm, shellInput, shellSelect } from "./input" import { shellQuote } from "../../core/src/shell" import { uniq } from "es-toolkit" import { PLimitPromiseQueue } from "../../core/src/concurrency" -import { Project } from "../../core/src/ast" +import { Project } from "../../core/src/server/messages" import { createAzureTokenResolver } from "./azuretoken" import { createAzureContentSafetyClient, @@ -130,7 +130,7 @@ class ModelManager implements ModelService { } export class NodeHost implements RuntimeHost { - readonly config: HostConfiguration + private readonly dotEnvPath: string project: Project userState: any = {} models: ModelService @@ -153,7 +153,6 @@ export class NodeHost implements RuntimeHost { readonly azureServerlessToken: AzureTokenResolver constructor(config: HostConfiguration) { - this.config = config this.models = new ModelManager(this) this.azureToken = createAzureTokenResolver( "Azure", @@ -167,8 +166,12 @@ export class NodeHost implements RuntimeHost { ) } + async readConfig(): Promise { + return resolveGlobalConfiguration(this.dotEnvPath) + } + private async syncDotEnv() { - const { envFile } = this.config + const { envFile } = await this.readConfig() if (existsSync(envFile)) { if (resolve(envFile) !== resolve(DOT_ENV_FILENAME)) logVerbose(`.env: loading ${envFile}`) diff --git a/packages/cli/src/run.ts b/packages/cli/src/run.ts index ef8ef54a44..a91333d857 100644 --- a/packages/cli/src/run.ts +++ b/packages/cli/src/run.ts @@ -274,11 +274,11 @@ export async function runScript( toolFiles, }) if (jsSource) - prj.templates.push({ + prj.scripts.push({ id: scriptId, jsSource, }) - const script = prj.templates.find( + const script = prj.scripts.find( (t) => t.id === scriptId || (t.filename && diff --git a/packages/cli/src/scripts.ts b/packages/cli/src/scripts.ts index ae3c43d5c3..7f28899643 100644 --- a/packages/cli/src/scripts.ts +++ b/packages/cli/src/scripts.ts @@ -8,19 +8,25 @@ import { fixPromptDefinitions, createScript as coreCreateScript, } from "../../core/src/scripts" -import { logError, logInfo, logVerbose } from "../../core/src/util" +import { logInfo, logVerbose } from "../../core/src/util" import { runtimeHost } from "../../core/src/host" import { RUNTIME_ERROR_CODE } from "../../core/src/constants" +import { + collectFolders, + filterScripts, + ScriptFilterOptions, +} from "../../core/src/ast" /** * Lists all the scripts in the project. * Displays id, title, group, filename, and system status. * Generates this list by first building the project. */ -export async function listScripts() { +export async function listScripts(options?: ScriptFilterOptions) { const prj = await buildProject() // Build the project to get script templates + const scripts = filterScripts(prj.scripts, options) // Filter scripts based on options console.log("id, title, group, filename, system") - prj.templates.forEach((t) => + scripts.forEach((t) => console.log( `${t.id}, ${t.title}, ${t.group || ""}, ${t.filename || "builtin"}, ${ t.isSystem ? "system" : "user" @@ -61,9 +67,9 @@ export async function fixScripts() { export async function compileScript(folders: string[]) { const project = await buildProject() // Build the project to gather script information await fixPromptDefinitions(project) // Fix prompt definitions before compiling - const scriptFolders = project.folders() // Retrieve available script folders + const scriptFolders = collectFolders(project) // Retrieve available script folders const foldersToCompile = ( - folders?.length ? folders : project.folders().map((f) => f.dirname) + folders?.length ? folders : scriptFolders.map((f) => f.dirname) ) .map((f) => scriptFolders.find((sf) => sf.dirname === f)) .filter((f) => f) diff --git a/packages/cli/src/server.ts b/packages/cli/src/server.ts index 2183e23aaf..a4a87bca5e 100644 --- a/packages/cli/src/server.ts +++ b/packages/cli/src/server.ts @@ -34,6 +34,7 @@ import { ChatChunk, ChatCancel, LanguageModelConfigurationResponse, + promptScriptListResponse, } from "../../core/src/server/messages" import { envInfo } from "./info" import { LanguageModel } from "../../core/src/chat" @@ -43,6 +44,7 @@ import { CreateChatCompletionRequest, } from "../../core/src/chattypes" import { randomHex } from "../../core/src/crypto" +import { buildProject } from "./build" /** * Starts a WebSocket server for handling chat and script execution. @@ -241,6 +243,19 @@ export async function startServer(options: { port: string; apiKey?: string }) { } break } + case "script.list": { + logVerbose(`project: list scripts`) + const project = await buildProject() + logVerbose( + `project: found ${project?.scripts?.length || 0} scripts` + ) + response = { + ok: true, + status: 0, + project, + } + break + } // Handle test run request case "tests.run": { logVerbose( diff --git a/packages/cli/src/test.ts b/packages/cli/src/test.ts index 71cd65dbce..39d0850876 100644 --- a/packages/cli/src/test.ts +++ b/packages/cli/src/test.ts @@ -44,6 +44,7 @@ import { ModelConnectionInfo, resolveModelConnectionInfo, } from "../../core/src/models" +import { filterScripts } from "../../core/src/ast" /** * Parses model specifications from a string and returns a ModelOptions object. @@ -255,12 +256,11 @@ export async function runPromptScriptTests( * @returns A Promise resolving to an array of filtered scripts. */ async function listTests(options: { ids?: string[]; groups?: string[] }) { - const { ids, groups } = options || {} const prj = await buildProject() - const scripts = prj.templates - .filter((t) => arrayify(t.tests)?.length) - .filter((t) => !ids?.length || ids.includes(t.id)) - .filter((t) => tagFilter(groups, t.group)) + const scripts = filterScripts(prj.scripts, { + ...(options || {}), + test: true, + }) return scripts } diff --git a/packages/core/src/ast.ts b/packages/core/src/ast.ts index 5501a919ae..4b7a4f3600 100644 --- a/packages/core/src/ast.ts +++ b/packages/core/src/ast.ts @@ -6,10 +6,8 @@ import { PROMPTY_REGEX, } from "./constants" import { host } from "./host" - -// Type alias for PromptScript used globally -type PromptScript = globalThis.PromptScript -export type { PromptScript } +import { Project } from "./server/messages" +import { arrayify, tagFilter } from "./util" // Interface representing a file reference, with a name and filename property export interface FileReference { @@ -57,43 +55,49 @@ export const eolPosition = 0x3fffffff // End of line position, a large constant export const eofPosition: CharPosition = [0x3fffffff, 0] // End of file position, a tuple with a large constant /** - * Represents a project containing templates and diagnostics. - * Provides utility methods to manage templates and diagnose issues. + * Groups templates by their directory and determines if JS or TS files are present. + * Useful for organizing templates based on their file type. + * @returns An array of folder information objects, each containing directory name and file type presence. */ -export class Project { - readonly templates: PromptScript[] = [] // Array of templates within the project - readonly diagnostics: Diagnostic[] = [] // Array of diagnostic records +export function collectFolders(prj: Project) { + const folders: Record< + string, + { dirname: string; js?: boolean; ts?: boolean } + > = {} + for (const t of Object.values(prj.scripts).filter( + // must have a filename and not propmty + (t) => t.filename && !PROMPTY_REGEX.test(t.filename) + )) { + const dirname = host.path.dirname(t.filename) // Get directory name from the filename + const folder = folders[dirname] || (folders[dirname] = { dirname }) + folder.js = folder.js || GENAI_ANYJS_REGEX.test(t.filename) // Check for presence of JS files + folder.ts = folder.ts || GENAI_ANYTS_REGEX.test(t.filename) // Check for presence of TS files + } + return Object.values(folders) // Return an array of folders with their properties +} - _finalizers: (() => void)[] = [] // Array of cleanup functions to be called when project is disposed +/** + * Retrieves a template by its ID. + * @param id - The ID of the template to retrieve. + * @returns The matching PromptScript or undefined if no match is found. + */ +export function resolveScript(prj: Project, id: string) { + return prj?.scripts?.find((t) => t.id == id) // Find and return the template with the matching ID +} - /** - * Groups templates by their directory and determines if JS or TS files are present. - * Useful for organizing templates based on their file type. - * @returns An array of folder information objects, each containing directory name and file type presence. - */ - folders() { - const folders: Record< - string, - { dirname: string; js?: boolean; ts?: boolean } - > = {} - for (const t of Object.values(this.templates).filter( - // must have a filename and not propmty - (t) => t.filename && !PROMPTY_REGEX.test(t.filename) - )) { - const dirname = host.path.dirname(t.filename) // Get directory name from the filename - const folder = folders[dirname] || (folders[dirname] = { dirname }) - folder.js = folder.js || GENAI_ANYJS_REGEX.test(t.filename) // Check for presence of JS files - folder.ts = folder.ts || GENAI_ANYTS_REGEX.test(t.filename) // Check for presence of TS files - } - return Object.values(folders) // Return an array of folders with their properties - } +export interface ScriptFilterOptions { + ids?: string[] + groups?: string[] + test?: boolean +} - /** - * Retrieves a template by its ID. - * @param id - The ID of the template to retrieve. - * @returns The matching PromptScript or undefined if no match is found. - */ - getTemplate(id: string) { - return this.templates.find((t) => t.id == id) // Find and return the template with the matching ID - } +export function filterScripts( + scripts: PromptScript[], + options: ScriptFilterOptions +) { + const { ids, groups, test } = options || {} + return scripts + .filter((t) => !test || arrayify(t.tests)?.length) + .filter((t) => !ids?.length || ids.includes(t.id)) + .filter((t) => tagFilter(groups, t.group)) } diff --git a/packages/core/src/config.ts b/packages/core/src/config.ts index f4ccf7297a..067752d9ee 100644 --- a/packages/core/src/config.ts +++ b/packages/core/src/config.ts @@ -40,7 +40,9 @@ export async function resolveGlobalConfiguration( }, }) if (validation.schemaError) - throw new Error(validation.schemaError) + throw new Error( + `Configuration error: ` + validation.schemaError + ) config = structuralMerge(config, parsed) } } diff --git a/packages/core/src/copy.ts b/packages/core/src/copy.ts index b9fffa88c1..27da4594ec 100644 --- a/packages/core/src/copy.ts +++ b/packages/core/src/copy.ts @@ -2,7 +2,6 @@ // including constructing file paths and handling copy operations, // with optional forking functionality. -import { PromptScript } from "./ast" // Import PromptScript type import { GENAI_MJS_EXT, GENAI_SRC } from "./constants" // Import constants for file extensions and source directory import { host } from "./host" // Import host module for file operations import { fileExists, writeText } from "./fs" // Import file system utilities diff --git a/packages/core/src/csv.ts b/packages/core/src/csv.ts index fb6df895ee..ef1db54069 100644 --- a/packages/core/src/csv.ts +++ b/packages/core/src/csv.ts @@ -68,6 +68,7 @@ export function CSVTryParse( ): object[] | undefined { const { trace } = options || {} try { + if (!text) return [] // Return empty array if CSV is empty // Attempt to parse the CSV return CSVParse(text, options) } catch (e) { @@ -85,6 +86,7 @@ export function CSVTryParse( * @returns A string representing the CSV formatted data. */ export function CSVStringify(csv: object[], options?: CSVStringifyOptions) { + if (!csv) return "" // Return empty string if CSV is empty // Convert objects to CSV string using the provided options return stringify(csv, options) } diff --git a/packages/core/src/expander.ts b/packages/core/src/expander.ts index bd6020f9bd..a2f711523f 100644 --- a/packages/core/src/expander.ts +++ b/packages/core/src/expander.ts @@ -1,4 +1,4 @@ -import { Project, PromptScript } from "./ast" +import { resolveScript } from "./ast" import { assert, normalizeFloat, normalizeInt } from "./util" import { MarkdownTrace } from "./trace" import { errorMessage, isCancelError, NotSupportedError } from "./error" @@ -30,6 +30,7 @@ import { resolveSystems } from "./systems" import { GenerationOptions, GenerationStatus } from "./generation" import { AICIRequest, ChatCompletionMessageParam } from "./chattypes" import { promptParametersSchemaToJSONSchema } from "./parameters" +import { Project } from "./server/messages" export async function callExpander( prj: Project, @@ -182,7 +183,7 @@ export async function expandTemplate( options.lineNumbers ?? template.lineNumbers ?? resolveSystems(prj, template, undefined, options) - .map((s) => prj.getTemplate(s)) + .map((s) => resolveScript(prj, s)) .some((t) => t?.lineNumbers) const temperature = options.temperature ?? @@ -242,7 +243,7 @@ export async function expandTemplate( const chatParticipants = prompt.chatParticipants.slice(0) const fileOutputs = prompt.fileOutputs.slice(0) const prediction = prompt.prediction - + if (prompt.logs?.length) trace.details("📝 console.log", prompt.logs) if (prompt.aici) trace.fence(prompt.aici, "yaml") trace.endDetails() @@ -283,7 +284,7 @@ export async function expandTemplate( messages, } - const system = prj.getTemplate(systems[i]) + const system = resolveScript(prj, systems[i]) if (!system) throw new Error(`system template ${systems[i]} not found`) diff --git a/packages/core/src/host.ts b/packages/core/src/host.ts index 23053fca13..67d1e9b0cd 100644 --- a/packages/core/src/host.ts +++ b/packages/core/src/host.ts @@ -2,7 +2,7 @@ import { CancellationToken } from "./cancellation" import { LanguageModel } from "./chat" import { Progress } from "./progress" import { AbortSignalOptions, MarkdownTrace, TraceOptions } from "./trace" -import { Project } from "./ast" +import { Project } from "./server/messages" import { HostConfiguration } from "./hostconfiguration" // this is typically an instance of TextDecoder @@ -160,12 +160,12 @@ export interface Host { } export interface RuntimeHost extends Host { - readonly config: HostConfiguration project: Project models: ModelService workspace: Omit azureToken: AzureTokenResolver + readConfig(): Promise readSecret(name: string): Promise // executes a process exec( diff --git a/packages/core/src/parameters.ts b/packages/core/src/parameters.ts index 095e1466d1..d721f42b56 100644 --- a/packages/core/src/parameters.ts +++ b/packages/core/src/parameters.ts @@ -1,10 +1,11 @@ import { rest } from "es-toolkit" -import { Project } from "./ast" +import { resolveScript } from "./ast" import { NotSupportedError } from "./error" import { isJSONSchema } from "./schema" import { resolveSystems } from "./systems" import { logError, normalizeFloat, normalizeInt } from "./util" import { YAMLStringify } from "./yaml" +import { Project } from "./server/messages" function isPromptParameterTypeRequired(t: PromptParameterType): boolean { const ta = t as any @@ -93,7 +94,7 @@ export function parsePromptParameters( ...(script.parameters || {}), } for (const system of resolveSystems(prj, script) - .map((s) => prj.getTemplate(s)) + .map((s) => resolveScript(prj, s)) .filter((t) => t?.parameters)) { Object.entries(system.parameters).forEach(([k, v]) => { parameters[`${system.id}.${k}`] = v diff --git a/packages/core/src/parser.ts b/packages/core/src/parser.ts index 6409ba57ac..12d8a8d927 100644 --- a/packages/core/src/parser.ts +++ b/packages/core/src/parser.ts @@ -1,6 +1,5 @@ // Importing utility functions and constants from other files import { logVerbose, strcmp } from "./util" // String comparison function -import { Project, PromptScript } from "./ast" // Class imports import { defaultPrompts } from "./default_prompts" // Default prompt data import { parsePromptScript } from "./template" // Function to parse scripts import { readText } from "./fs" // Function to read text from a file @@ -10,6 +9,8 @@ import { PDF_MIME_TYPE, XLSX_MIME_TYPE, } from "./constants" // Constants for MIME types and prefixes +import { diag } from "mathjs" +import { Project } from "./server/messages" /** * Converts a string to a character position represented as [row, column]. @@ -76,16 +77,10 @@ const BINARY_MIME_TYPES = [ */ export async function parseProject(options: { scriptFiles: string[] }) { const { scriptFiles } = options - const prj = new Project() // Initialize a new project instance - - // Helper function to run finalizers stored in the project - const runFinalizers = () => { - const fins = prj._finalizers.slice() // Copy finalizers - prj._finalizers = [] // Clear the finalizers - for (const fin of fins) fin() // Execute each finalizer - } - - runFinalizers() // Run any initial finalizers + const prj: Project = { + scripts: [], + diagnostics: [], + } // Initialize a new project instance // Clone the default prompts const deflPr: Record = Object.assign({}, defaultPrompts) @@ -98,16 +93,14 @@ export async function parseProject(options: { scriptFiles: string[] }) { continue } // Skip if no template is parsed delete deflPr[tmpl.id] // Remove the parsed template from defaults - prj.templates.push(tmpl) // Add to project templates + prj.scripts.push(tmpl) // Add to project templates } // Add remaining default prompts to the project for (const [id, v] of Object.entries(deflPr)) { - prj.templates.push(await parsePromptScript(BUILTIN_PREFIX + id, v, prj)) + prj.scripts.push(await parsePromptScript(BUILTIN_PREFIX + id, v, prj)) } - runFinalizers() // Run finalizers after processing all scripts - /** * Generates a sorting key for a PromptScript * Determines priority based on whether a script is unlisted or has a filename. @@ -120,7 +113,7 @@ export async function parseProject(options: { scriptFiles: string[] }) { } // Sort templates by the generated key - prj.templates.sort((a, b) => strcmp(templKey(a), templKey(b))) + prj.scripts.sort((a, b) => strcmp(templKey(a), templKey(b))) return prj // Return the fully parsed project } diff --git a/packages/core/src/promptcontext.ts b/packages/core/src/promptcontext.ts index 8affe362c6..63e1a9b649 100644 --- a/packages/core/src/promptcontext.ts +++ b/packages/core/src/promptcontext.ts @@ -17,7 +17,7 @@ import { fuzzSearch } from "./fuzzsearch" import { grepSearch } from "./grep" import { resolveFileContents, toWorkspaceFile } from "./file" import { vectorSearch } from "./vectorsearch" -import { Project } from "./ast" +import { Project } from "./server/messages" import { shellParse } from "./shell" import { PLimitPromiseQueue } from "./concurrency" import { NotSupportedError } from "./error" diff --git a/packages/core/src/promptrunner.ts b/packages/core/src/promptrunner.ts index ebf2976069..a0233030a4 100644 --- a/packages/core/src/promptrunner.ts +++ b/packages/core/src/promptrunner.ts @@ -1,6 +1,6 @@ // Import necessary modules and functions for handling chat sessions, templates, file management, etc. import { executeChatSession, tracePromptResult } from "./chat" -import { Project, PromptScript } from "./ast" +import { Project } from "./server/messages" import { stringToPos } from "./parser" import { arrayify, assert, logError, logVerbose, relativePath } from "./util" import { runtimeHost } from "./host" diff --git a/packages/core/src/runpromptcontext.ts b/packages/core/src/runpromptcontext.ts index 7a4939a035..599b67e0dd 100644 --- a/packages/core/src/runpromptcontext.ts +++ b/packages/core/src/runpromptcontext.ts @@ -68,12 +68,13 @@ import { } from "./error" import { resolveLanguageModel } from "./lm" import { concurrentLimit } from "./concurrency" -import { Project } from "./ast" +import { resolveScript } from "./ast" import { dedent } from "./indent" import { runtimeHost } from "./host" import { writeFileEdits } from "./fileedits" import { agentAddMemory, agentQueryMemory } from "./agent" import { YAMLStringify } from "./yaml" +import { Project } from "./server/messages" export function createChatTurnGenerationContext( options: GenerationOptions, @@ -164,7 +165,7 @@ export function createChatTurnGenerationContext( role: (r) => { current.role = r return res - } + }, }) return res }, @@ -684,7 +685,7 @@ export function createChatGenerationContext( for (const systemId of systemScripts) { checkCancelled(cancellationToken) - const system = prj.getTemplate(systemId) + const system = resolveScript(prj, systemId) if (!system) throw new Error( `system template ${systemId} not found` diff --git a/packages/core/src/schemas/hostconfiguration.schema.json b/packages/core/src/schemas/hostconfiguration.schema.json new file mode 100644 index 0000000000..e7a6bee13a --- /dev/null +++ b/packages/core/src/schemas/hostconfiguration.schema.json @@ -0,0 +1,18 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "title": "GenAIScript Configuration", + "type": "object", + "properties": { + "envFile": { + "type": "string", + "description": "Path to the .env file" + }, + "include": { + "type": "array", + "items": { + "type": "string" + }, + "description": "List of glob paths to scan for genai scripts" + } + } +} diff --git a/packages/core/src/scripts.ts b/packages/core/src/scripts.ts index 2926d6b483..b9dab61354 100644 --- a/packages/core/src/scripts.ts +++ b/packages/core/src/scripts.ts @@ -1,10 +1,11 @@ -import { Project } from "./ast" +import { collectFolders } from "./ast" import { NEW_SCRIPT_TEMPLATE } from "./constants" import { promptDefinitions } from "./default_prompts" import { tryReadText, writeText } from "./fs" import { host } from "./host" import { logVerbose } from "./util" import { dedent } from "./indent" +import { Project } from "./server/messages" export function createScript( name: string, @@ -24,8 +25,8 @@ export function createScript( } export async function fixPromptDefinitions(project: Project) { - const folders = project.folders() - const systems = project.templates.filter((t) => t.isSystem) + const folders = collectFolders(project) + const systems = project.scripts.filter((t) => t.isSystem) const tools = systems.map(({ defTools }) => defTools || []).flat() for (const folder of folders) { diff --git a/packages/core/src/server/client.ts b/packages/core/src/server/client.ts index a6f66fac80..913a90e12c 100644 --- a/packages/core/src/server/client.ts +++ b/packages/core/src/server/client.ts @@ -29,6 +29,9 @@ import { ChatChunk, ChatStart, LanguageModelConfigurationRequest, + Project, + PromptScriptList, + promptScriptListResponse, } from "./messages" export type LanguageModelChatRequest = ( @@ -264,6 +267,12 @@ export class WebSocketClient extends EventTarget { return res.response } + async listScripts(): Promise { + const res = await this.queue({ type: "script.list" }) + const project = (res.response as promptScriptListResponse)?.project + return project + } + async startScript( script: string, files: string[], diff --git a/packages/core/src/server/messages.ts b/packages/core/src/server/messages.ts index 4b883ebf4a..035aa4c2d0 100644 --- a/packages/core/src/server/messages.ts +++ b/packages/core/src/server/messages.ts @@ -2,6 +2,15 @@ import { ChatCompletionAssistantMessageParam } from "../chattypes" import { GenerationResult } from "../generation" import { LanguageModelConfiguration, ResponseStatus } from "../host" +/** + * Represents a project containing templates and diagnostics. + * Provides utility methods to manage templates and diagnose issues. + */ +export interface Project { + scripts: PromptScript[] // Array of templates within the project + diagnostics: Diagnostic[] // Array of diagnostic records +} + export interface RequestMessage { type: string id: string @@ -94,6 +103,14 @@ export interface PromptScriptRunOptions { varsMap?: Record } +export interface PromptScriptList extends RequestMessage { + type: "script.list" +} + +export interface promptScriptListResponse extends ResponseStatus { + project: Project +} + export interface PromptScriptStart extends RequestMessage { type: "script.start" runId: string @@ -193,6 +210,7 @@ export type RequestMessages = | PromptScriptAbort | ChatChunk | LanguageModelConfigurationRequest + | PromptScriptList export type PromptScriptResponseEvents = | PromptScriptProgressResponseEvent diff --git a/packages/core/src/systems.ts b/packages/core/src/systems.ts index 3c7e6c8189..8586cbd751 100644 --- a/packages/core/src/systems.ts +++ b/packages/core/src/systems.ts @@ -3,10 +3,10 @@ // It analyzes script options and the JavaScript source code to determine which systems to include or exclude. import { uniq } from "es-toolkit" -import { Project } from "./ast" import { arrayify } from "./util" import { GenerationOptions } from "./generation" import { isToolsSupported } from "./tools" +import { Project } from "./server/messages" /** * Function to resolve and return a list of systems based on the provided script and project. @@ -104,7 +104,7 @@ export function resolveSystems( * @returns An array of system IDs associated with the specified tool. */ function resolveSystemFromTools(prj: Project, tool: string): string[] { - const system = prj.templates.filter( + const system = prj.scripts.filter( (t) => t.isSystem && t.defTools?.find((to) => to.id.startsWith(tool)) ) const res = system.map(({ id }) => id) @@ -126,7 +126,7 @@ export function resolveTools( systems: string[], tools: string[] ): { id: string; description: string }[] { - const { templates: scripts } = prj + const { scripts: scripts } = prj const toolScripts = uniq([ ...systems.map((sid) => scripts.find((s) => s.id === sid)), ...tools.map((tid) => diff --git a/packages/core/src/template.ts b/packages/core/src/template.ts index c21a98834a..c851d8b44f 100644 --- a/packages/core/src/template.ts +++ b/packages/core/src/template.ts @@ -4,7 +4,7 @@ * data types and formats. */ -import { Project, PromptScript } from "./ast" +import { Project } from "./server/messages" import { BUILTIN_PREFIX, GENAI_ANY_REGEX, PROMPTY_REGEX } from "./constants" import { errorMessage } from "./error" import { host } from "./host" @@ -12,7 +12,6 @@ import { JSON5TryParse } from "./json5" import { humanize } from "inflection" import { validateSchema } from "./schema" import { promptyParse, promptyToGenAIScript } from "./prompty" -import { kind } from "openai/_shims/index.mjs" /** * Extracts a template ID from the given filename by removing specific extensions @@ -87,14 +86,14 @@ class Checker { /** * Constructs a new Checker instance. * - * @param template - The prompt-like object to validate. + * @param script - The prompt-like object to validate. * @param filename - The filename of the script. * @param diagnostics - The diagnostics array to report errors to. * @param js - The JavaScript source code of the script. * @param jsobj - The parsed JSON object of the script. */ constructor( - public template: T, + public script: T, public filename: string, public diagnostics: Diagnostic[], public js: string, @@ -359,8 +358,7 @@ function parsePromptScriptTools(jsSource: string) { async function parsePromptTemplateCore( filename: string, content: string, - prj: Project, - finalizer: (checker: Checker) => void + prj: Project ) { const r = { id: templateIdFromFileName(filename), @@ -381,8 +379,50 @@ async function parsePromptTemplateCore( content, meta ) - prj._finalizers.push(() => finalizer(checker)) - return checker.template + const obj = checker.validateKV(() => { + // Validate various fields using the Checker methods + checker.checkString("title") + checker.checkString("description") + checker.checkString("model") + checker.checkString("responseType") + checker.checkJSONSchema("responseSchema") + + checker.checkString("embeddingsModel") + + checker.checkBool("unlisted") + + checker.checkNat("maxTokens") + checker.checkNumber("temperature") + checker.checkNumber("topP") + checker.checkNumber("seed") + checker.checkNat("flexTokens") + + checker.checkStringArray("system") + checker.checkStringArray("excludedSystem") + checker.checkStringArray("files") + checker.checkString("group") + + checker.checkBool("isSystem") + checker.checkRecord("parameters") + checker.checkRecord("vars") + checker.checkStringArray("secrets") + + checker.checkBool("lineNumbers") + checker.checkObjectOrObjectArray("tests") + checker.checkStringArray("tools") + checker.checkStringOrBool("cache") + checker.checkString("cacheName") + checker.checkString("filename") + checker.checkString("contentSafety") + checker.checkStringArray("choices") + checker.checkNumber("topLogprobs") + + checker.checkRecord("modelConcurrency") + checker.checkObjectArray("defTools") + checker.checkBool("logprobs") + }) + Object.assign(checker.script, obj) + return checker.script } catch (e) { prj.diagnostics.push({ filename, @@ -415,51 +455,5 @@ export async function parsePromptScript( content = await promptyToGenAIScript(doc) } - return await parsePromptTemplateCore(filename, content, prj, (c) => { - const obj = c.validateKV(() => { - // Validate various fields using the Checker methods - c.checkString("title") - c.checkString("description") - c.checkString("model") - c.checkString("responseType") - c.checkJSONSchema("responseSchema") - - c.checkString("embeddingsModel") - - c.checkBool("unlisted") - - c.checkNat("maxTokens") - c.checkNumber("temperature") - c.checkNumber("topP") - c.checkNumber("seed") - c.checkNat("flexTokens") - - c.checkStringArray("system") - c.checkStringArray("excludedSystem") - c.checkStringArray("files") - c.checkString("group") - - c.checkBool("isSystem") - c.checkRecord("parameters") - c.checkRecord("vars") - c.checkStringArray("secrets") - - c.checkBool("lineNumbers") - c.checkObjectOrObjectArray("tests") - c.checkStringArray("tools") - c.checkStringOrBool("cache") - c.checkString("cacheName") - c.checkString("filename") - c.checkString("contentSafety") - c.checkStringArray("choices") - c.checkNumber("topLogprobs") - - c.checkRecord("modelConcurrency") - c.checkObjectArray("defTools") - c.checkBool("logprobs") - }) - - const r = c.template - Object.assign(r, obj) - }) + return await parsePromptTemplateCore(filename, content, prj) } diff --git a/packages/core/src/testhost.ts b/packages/core/src/testhost.ts index ecae750b57..e09a601a2a 100644 --- a/packages/core/src/testhost.ts +++ b/packages/core/src/testhost.ts @@ -35,9 +35,9 @@ import { isAbsolute, } from "node:path" import { LanguageModel } from "./chat" -import { Project } from "./ast" import { NotSupportedError } from "./error" import { HostConfiguration } from "./hostconfiguration" +import { Project } from "./server/messages" // Function to create a frozen object representing Node.js path methods // This object provides utility methods for path manipulations @@ -56,7 +56,6 @@ export function createNodePath(): Path { // Class representing a test host for runtime, implementing the RuntimeHost interface export class TestHost implements RuntimeHost { - config: HostConfiguration project: Project // State object to store user-specific data userState: any = {} @@ -87,6 +86,10 @@ export class TestHost implements RuntimeHost { setRuntimeHost(new TestHost()) } + async readConfig() { + return {} + } + contentSafety( id?: "azure", options?: TraceOptions diff --git a/packages/core/src/util.ts b/packages/core/src/util.ts index 8d05c3a17e..43fd81cff6 100644 --- a/packages/core/src/util.ts +++ b/packages/core/src/util.ts @@ -285,16 +285,16 @@ export function renderWithPrecision( } export function tagFilter(tags: string[], tag: string) { - if (!tags?.length || !tag) return true - const ltag = tag.toLocaleLowerCase() - let inclusive = false + if (!tags?.length) return true + const ltag = tag?.toLocaleLowerCase() || "" + let exclusive = false for (const t of tags) { const lt = t.toLocaleLowerCase() const exclude = lt.startsWith(":!") - if (!exclude) inclusive = true + if (exclude) exclusive = true if (exclude && ltag.startsWith(lt.slice(2))) return false else if (ltag.startsWith(t)) return true } - return !inclusive + return exclusive } diff --git a/packages/sample/genaiscript.config.json b/packages/sample/genaiscript.config.json new file mode 100644 index 0000000000..9630d91ec2 --- /dev/null +++ b/packages/sample/genaiscript.config.json @@ -0,0 +1,4 @@ +{ + "$schema": "../../core/src/schemas/hostconfiguration.schema.json", + "include": ["../../genaisrc/*.genai.mts"] +} diff --git a/packages/vscode/src/chatparticipant.ts b/packages/vscode/src/chatparticipant.ts index 5cf72f94ba..aa2bdfeb08 100644 --- a/packages/vscode/src/chatparticipant.ts +++ b/packages/vscode/src/chatparticipant.ts @@ -60,7 +60,7 @@ export async function activateChatParticipant(state: ExtensionState) { } const { project } = state - const templates = project.templates + const templates = project.scripts .filter((s) => !s.isSystem && !s.unlisted) .sort((a, b) => a.id.localeCompare(b.id)) diff --git a/packages/vscode/src/fragmentcommands.ts b/packages/vscode/src/fragmentcommands.ts index 69e7b47f53..ceca1f862b 100644 --- a/packages/vscode/src/fragmentcommands.ts +++ b/packages/vscode/src/fragmentcommands.ts @@ -77,7 +77,7 @@ export function activateFragmentCommands(state: ExtensionState) { filter?: (p: PromptScript) => boolean }) => { const { filter = () => true } = options || {} - const templates = state.project.templates + const templates = state.project.scripts .filter((t) => !t.isSystem && t.group !== "infrastructure") .filter(filter) @@ -138,7 +138,7 @@ export function activateFragmentCommands(state: ExtensionState) { fragment instanceof vscode.Uri && GENAI_ANY_REGEX.test(fragment.path) ) { - template = state.project.templates.find( + template = state.project.scripts.find( (p) => p.filename === (fragment as vscode.Uri).fsPath ) assert(template !== undefined) @@ -169,7 +169,7 @@ export function activateFragmentCommands(state: ExtensionState) { let template: PromptScript let files: vscode.Uri[] if (GENAI_ANY_REGEX.test(file.path)) { - template = state.project.templates.find( + template = state.project.scripts.find( (p) => p.filename === file.fsPath ) assert(template !== undefined) diff --git a/packages/vscode/src/promptcommands.ts b/packages/vscode/src/promptcommands.ts index 6efec3c84b..ffc6e9416b 100644 --- a/packages/vscode/src/promptcommands.ts +++ b/packages/vscode/src/promptcommands.ts @@ -40,7 +40,7 @@ export function activatePromptCommands(state: ExtensionState) { async (template: PromptScript | string) => { if (!template) { if (!state.project) await state.parseWorkspace() - const templates = state.project?.templates + const templates = state.project?.scripts if (!templates?.length) return const picked = await vscode.window.showQuickPick( templatesToQuickPickItems(templates), @@ -52,7 +52,7 @@ export function activatePromptCommands(state: ExtensionState) { template = picked.template } else if (typeof template === "string") { if (!state.project) await state.parseWorkspace() - template = state.project?.templates.find( + template = state.project?.scripts.find( (t) => t.id === template ) } diff --git a/packages/vscode/src/prompttree.ts b/packages/vscode/src/prompttree.ts index 4c5d1de23e..ad503a28ea 100644 --- a/packages/vscode/src/prompttree.ts +++ b/packages/vscode/src/prompttree.ts @@ -24,7 +24,13 @@ class PromptTreeDataProvider item.id = `genaiscript.promptCategory.${element}` return item } else { - const { id, filename, title, description = "" } = element + const { + id, + filename, + title, + description = "", + system = [], + } = element const ai = this.state.aiRequest const { computing, options, progress } = ai || {} const { template } = options || {} @@ -45,11 +51,11 @@ class PromptTreeDataProvider } item.tooltip = new vscode.MarkdownString( ` -## ${title} +## ${title} \`${id}\` ${description} -- id: \`${id}\` +- filename: \`${filename ? vscode.workspace.asRelativePath(filename) : "builtin"}\` `, true ) @@ -67,13 +73,14 @@ ${description} async getChildren( element?: PromptTreeNode | undefined ): Promise { - const templates = this.state.project?.templates || [] + const project = this.state.project + const templates = project?.scripts || [] if (!element) { - // collect and sort all groups - const cats = Object.keys(groupBy(templates, templateGroup)) - return [...cats.filter((t) => t !== "system"), "system"] + if (!templates.length) return [] + const groups = Object.keys(groupBy(templates, templateGroup)) + return [...groups.filter((t) => t !== "system"), "system"] } else if (typeof element === "string") { - const templates = this.state.project?.templates || [] + const templates = this.state.project?.scripts || [] return templates.filter((t) => templateGroup(t) === element) } else { return undefined diff --git a/packages/vscode/src/state.ts b/packages/vscode/src/state.ts index 3da3108b70..7dfc3fcb28 100644 --- a/packages/vscode/src/state.ts +++ b/packages/vscode/src/state.ts @@ -4,12 +4,12 @@ import { ExtensionContext } from "vscode" import { VSCodeHost } from "./vshost" import { applyEdits, toRange } from "./edit" import { Utils } from "vscode-uri" -import { findFiles, listFiles, saveAllTextDocuments } from "./fs" +import { listFiles, saveAllTextDocuments } from "./fs" import { startLocalAI } from "./localai" import { hasOutputOrTraceOpened } from "./markdowndocumentprovider" import { pickLanguageModel } from "./lmaccess" import { parseAnnotations } from "../../core/src/annotations" -import { Project } from "../../core/src/ast" +import { Project } from "../../core/src/server/messages" import { JSONLineCache } from "../../core/src/cache" import { ChatCompletionsProgressReport } from "../../core/src/chattypes" import { fixPromptDefinitions } from "../../core/src/scripts" @@ -19,16 +19,13 @@ import { CHANGE, AI_REQUESTS_CACHE, TOOL_ID, - GENAI_ANYJS_GLOB, } from "../../core/src/constants" import { isCancelError } from "../../core/src/error" import { resolveModelConnectionInfo } from "../../core/src/models" -import { parseProject } from "../../core/src/parser" import { MarkdownTrace } from "../../core/src/trace" import { logInfo, groupBy } from "../../core/src/util" import { CORE_VERSION } from "../../core/src/version" import { Fragment, GenerationResult } from "../../core/src/generation" -import { parametersToVars } from "../../core/src/parameters" import { hash, randomHex } from "../../core/src/crypto" import { delay } from "es-toolkit" @@ -368,6 +365,8 @@ export class ExtensionState extends EventTarget { } get project() { + if (!this._project) + this.parseWorkspace() return this._project } @@ -388,9 +387,6 @@ export class ExtensionState extends EventTarget { async activate() { await this.host.activate() - await this.parseWorkspace() - await this.fixPromptDefinitions() - logInfo("genaiscript extension activated") } @@ -399,11 +395,6 @@ export class ExtensionState extends EventTarget { if (project) await fixPromptDefinitions(project) } - async findScripts() { - const scriptFiles = await findFiles(GENAI_ANYJS_GLOB) - return scriptFiles - } - async parseWorkspace() { if (this._parseWorkspacePromise) return this._parseWorkspacePromise @@ -414,11 +405,7 @@ export class ExtensionState extends EventTarget { await saveAllTextDocuments() performance.mark(`project-start`) performance.mark(`scan-tools`) - const scriptFiles = await this.findScripts() - performance.mark(`parse-project`) - const newProject = await parseProject({ - scriptFiles, - }) + const newProject = await this.host.server.client.listScripts() await this.setProject(newProject) this.setDiagnostics() logMeasure(`project`, `project-start`, `project-end`) diff --git a/packages/vscode/src/taskprovider.ts b/packages/vscode/src/taskprovider.ts index 2827fb252c..2811f747be 100644 --- a/packages/vscode/src/taskprovider.ts +++ b/packages/vscode/src/taskprovider.ts @@ -18,7 +18,7 @@ export async function activeTaskProvider(state: ExtensionState) { const exeArgs = cliPath ? [] : ["--yes", `genaiscript@${cliVersion}`] - const scripts = state.project.templates.filter( + const scripts = state.project.scripts.filter( (t) => !t.isSystem && t.group !== "infrastructure" ) const tasks = scripts.map((script) => { diff --git a/packages/vscode/src/testcontroller.ts b/packages/vscode/src/testcontroller.ts index 6f4366590f..3aaf6d4bbf 100644 --- a/packages/vscode/src/testcontroller.ts +++ b/packages/vscode/src/testcontroller.ts @@ -35,7 +35,7 @@ export async function activateTestController(state: ExtensionState) { if (!vscode.workspace.workspaceFolders) return // handle the case of no open folders if (testToResolve) { - const script = state.project.templates.find( + const script = state.project.scripts.find( (script) => vscode.workspace.asRelativePath(script.filename) === vscode.workspace.asRelativePath(testToResolve.uri) @@ -51,7 +51,7 @@ export async function activateTestController(state: ExtensionState) { if (!state.project) await state.parseWorkspace() if (token?.isCancellationRequested) return const scripts = - state.project.templates.filter((t) => arrayify(t.tests)?.length) || + state.project.scripts.filter((t) => arrayify(t.tests)?.length) || [] // refresh existing for (const script of scripts) { @@ -86,7 +86,7 @@ export async function activateTestController(state: ExtensionState) { const scripts = Array.from(tests) .map((test) => ({ test, - script: project.templates.find((s) => s.id === test.id), + script: project.scripts.find((s) => s.id === test.id), })) .filter(({ script }) => script)