Use CLI to list scripts (#893)

* templates to scripts

* removing project class

* fix: 🐛 return empty array for empty CSV input

* refactor: ♻️ migrate Project to server/messages module

* get scripts from cli

* feat: add group filtering and schema validation 📝

* always read config when needed

* fix: 🐛 return empty string if CSV is empty

docs/src/content/docs/reference/cli/commands.md

@@ -98,7 +98,7 @@ Options:
   -v, --verbose                       verbose output
   -pv, --promptfoo-version [version]  promptfoo version, default is 0.97.0
   -os, --out-summary <file>           append output summary in file
-  --groups <groups...>                groups to include or exclude. Use :!
+  -g, --groups <groups...>            groups to include or exclude. Use :!
                                       prefix to exclude
   -h, --help                          display help for command
 ```
@@ -111,8 +111,9 @@ Usage: genaiscript test list [options]
 List available tests in workspace
 
 Options:
-  --groups <groups...>  groups to include or exclude. Use :! prefix to exclude
-  -h, --help            display help for command
+  -g, --groups <groups...>  groups to include or exclude. Use :! prefix to
+                            exclude
+  -h, --help                display help for command
 ```
 
 ### `test view`
@@ -137,7 +138,7 @@ Options:
   -h, --help                display help for command
 
 Commands:
-  list                      List all available scripts in workspace
+  list [options]            List all available scripts in workspace
   create <name>             Create a new script
   fix                       fix all definition files
   compile [folders...]      Compile all scripts in workspace
@@ -153,7 +154,9 @@ Usage: genaiscript scripts list [options]
 List all available scripts in workspace
 
 Options:
-  -h, --help  display help for command
+  -g, --groups <groups...>  groups to include or exclude. Use :! prefix to
+                            exclude
+  -h, --help                display help for command
 ```
 
 ### `scripts create`

genaisrc/blog-generator.genai.mts

@@ -13,6 +13,7 @@ script({
             description: "The topic and goal of the article",
         },
     },
+    group: "docs"
 })
 let { topic, theme } = env.vars
 

genaisrc/docs-referencer.genai.mts

@@ -9,6 +9,7 @@ script({
     },
     system: ["system", "system.files"],
     tools: ["fs", "md"],
+    group: "docs"
 })
 
 const api = env.vars.api || "git"

genaisrc/docs-sample-generator.genai.mts

@@ -9,6 +9,7 @@ script({
             default: "defTool",
         },
     },
+    group: "docs"
 })
 
 const api = env.vars.api + ""

packages/cli/src/build.ts

@@ -23,7 +23,8 @@ export async function buildProject(options?: {
     } else {
         let tps = arrayify(toolsPath)
         if (!tps?.length) {
-            tps = [GENAI_ANYJS_GLOB, ...arrayify(runtimeHost.config.include)]
+            const config = await runtimeHost.readConfig()
+            tps = [GENAI_ANYJS_GLOB, ...arrayify(config.include)]
         }
         tps = arrayify(tps)
         scriptFiles = []

packages/cli/src/cli.ts

@@ -218,7 +218,7 @@ export async function cli() {
         )
         .option("-os, --out-summary <file>", "append output summary in file")
         .option(
-            "--groups <groups...>",
+            "-g, --groups <groups...>",
             "groups to include or exclude. Use :! prefix to exclude"
         )
         .action(scriptsTest) // Action to run the tests
@@ -228,7 +228,7 @@ export async function cli() {
         .description("List available tests in workspace")
         .action(scriptTestList) // Action to list the tests
         .option(
-            "--groups <groups...>",
+            "-g, --groups <groups...>",
             "groups to include or exclude. Use :! prefix to exclude"
         )
 
@@ -245,6 +245,10 @@ export async function cli() {
     scripts
         .command("list", { isDefault: true })
         .description("List all available scripts in workspace")
+        .option(
+            "-g, --groups <groups...>",
+            "groups to include or exclude. Use :! prefix to exclude"
+        )
         .action(listScripts) // Action to list scripts
     scripts
         .command("create")

packages/cli/src/info.ts

@@ -37,8 +37,9 @@ export async function envInfo(
     options?: { token?: boolean; error?: boolean }
 ) {
     const { token, error } = options || {}
-    const res: any = {}
-    res[".env"] = runtimeHost.config.envFile ?? ""
+    const config = await runtimeHost.readConfig()
+    const res: any = {}    
+    res[".env"] = config.envFile ?? ""
     res.providers = []
     const env = process.env
 
@@ -103,7 +104,7 @@ async function resolveScriptsConnectionInfo(
  */
 export async function modelInfo(script: string, options?: { token?: boolean }) {
     const prj = await buildProject()
-    const templates = prj.templates.filter(
+    const templates = prj.scripts.filter(
         (t) =>
             !script ||
             t.id === script ||

packages/cli/src/nodehost.ts

@@ -59,7 +59,7 @@ import { shellConfirm, shellInput, shellSelect } from "./input"
 import { shellQuote } from "../../core/src/shell"
 import { uniq } from "es-toolkit"
 import { PLimitPromiseQueue } from "../../core/src/concurrency"
-import { Project } from "../../core/src/ast"
+import { Project } from "../../core/src/server/messages"
 import { createAzureTokenResolver } from "./azuretoken"
 import {
     createAzureContentSafetyClient,
@@ -130,7 +130,7 @@ class ModelManager implements ModelService {
 }
 
 export class NodeHost implements RuntimeHost {
-    readonly config: HostConfiguration
+    private readonly dotEnvPath: string
     project: Project
     userState: any = {}
     models: ModelService
@@ -153,7 +153,6 @@ export class NodeHost implements RuntimeHost {
     readonly azureServerlessToken: AzureTokenResolver
 
     constructor(config: HostConfiguration) {
-        this.config = config
         this.models = new ModelManager(this)
         this.azureToken = createAzureTokenResolver(
             "Azure",
@@ -167,8 +166,12 @@ export class NodeHost implements RuntimeHost {
         )
     }
 
+    async readConfig(): Promise<HostConfiguration> {
+        return resolveGlobalConfiguration(this.dotEnvPath)
+    }
+
     private async syncDotEnv() {
-        const { envFile } = this.config
+        const { envFile } = await this.readConfig()
         if (existsSync(envFile)) {
             if (resolve(envFile) !== resolve(DOT_ENV_FILENAME))
                 logVerbose(`.env: loading ${envFile}`)

packages/cli/src/run.ts

@@ -274,11 +274,11 @@ export async function runScript(
         toolFiles,
     })
     if (jsSource)
-        prj.templates.push({
+        prj.scripts.push({
             id: scriptId,
             jsSource,
         })
-    const script = prj.templates.find(
+    const script = prj.scripts.find(
         (t) =>
             t.id === scriptId ||
             (t.filename &&

packages/cli/src/scripts.ts

@@ -8,19 +8,25 @@ import {
     fixPromptDefinitions,
     createScript as coreCreateScript,
 } from "../../core/src/scripts"
-import { logError, logInfo, logVerbose } from "../../core/src/util"
+import { logInfo, logVerbose } from "../../core/src/util"
 import { runtimeHost } from "../../core/src/host"
 import { RUNTIME_ERROR_CODE } from "../../core/src/constants"
+import {
+    collectFolders,
+    filterScripts,
+    ScriptFilterOptions,
+} from "../../core/src/ast"
 
 /**
  * Lists all the scripts in the project.
  * Displays id, title, group, filename, and system status.
  * Generates this list by first building the project.
  */
-export async function listScripts() {
+export async function listScripts(options?: ScriptFilterOptions) {
     const prj = await buildProject() // Build the project to get script templates
+    const scripts = filterScripts(prj.scripts, options) // Filter scripts based on options
     console.log("id, title, group, filename, system")
-    prj.templates.forEach((t) =>
+    scripts.forEach((t) =>
         console.log(
             `${t.id}, ${t.title}, ${t.group || ""}, ${t.filename || "builtin"}, ${
                 t.isSystem ? "system" : "user"
@@ -61,9 +67,9 @@ export async function fixScripts() {
 export async function compileScript(folders: string[]) {
     const project = await buildProject() // Build the project to gather script information
     await fixPromptDefinitions(project) // Fix prompt definitions before compiling
-    const scriptFolders = project.folders() // Retrieve available script folders
+    const scriptFolders = collectFolders(project) // Retrieve available script folders
     const foldersToCompile = (
-        folders?.length ? folders : project.folders().map((f) => f.dirname)
+        folders?.length ? folders : scriptFolders.map((f) => f.dirname)
     )
         .map((f) => scriptFolders.find((sf) => sf.dirname === f))
         .filter((f) => f)

packages/cli/src/server.ts

@@ -34,6 +34,7 @@ import {
     ChatChunk,
     ChatCancel,
     LanguageModelConfigurationResponse,
+    promptScriptListResponse,
 } from "../../core/src/server/messages"
 import { envInfo } from "./info"
 import { LanguageModel } from "../../core/src/chat"
@@ -43,6 +44,7 @@ import {
     CreateChatCompletionRequest,
 } from "../../core/src/chattypes"
 import { randomHex } from "../../core/src/crypto"
+import { buildProject } from "./build"
 
 /**
  * Starts a WebSocket server for handling chat and script execution.
@@ -241,6 +243,19 @@ export async function startServer(options: { port: string; apiKey?: string }) {
                         }
                         break
                     }
+                    case "script.list": {
+                        logVerbose(`project: list scripts`)
+                        const project = await buildProject()
+                        logVerbose(
+                            `project: found ${project?.scripts?.length || 0} scripts`
+                        )
+                        response = <promptScriptListResponse>{
+                            ok: true,
+                            status: 0,
+                            project,
+                        }
+                        break
+                    }
                     // Handle test run request
                     case "tests.run": {
                         logVerbose(

packages/cli/src/test.ts

@@ -44,6 +44,7 @@ import {
     ModelConnectionInfo,
     resolveModelConnectionInfo,
 } from "../../core/src/models"
+import { filterScripts } from "../../core/src/ast"
 
 /**
  * Parses model specifications from a string and returns a ModelOptions object.
@@ -255,12 +256,11 @@ export async function runPromptScriptTests(
  * @returns A Promise resolving to an array of filtered scripts.
  */
 async function listTests(options: { ids?: string[]; groups?: string[] }) {
-    const { ids, groups } = options || {}
     const prj = await buildProject()
-    const scripts = prj.templates
-        .filter((t) => arrayify(t.tests)?.length)
-        .filter((t) => !ids?.length || ids.includes(t.id))
-        .filter((t) => tagFilter(groups, t.group))
+    const scripts = filterScripts(prj.scripts, {
+        ...(options || {}),
+        test: true,
+    })
     return scripts
 }
 

packages/core/src/ast.ts

@@ -6,10 +6,8 @@ import {
     PROMPTY_REGEX,
 } from "./constants"
 import { host } from "./host"
-
-// Type alias for PromptScript used globally
-type PromptScript = globalThis.PromptScript
-export type { PromptScript }
+import { Project } from "./server/messages"
+import { arrayify, tagFilter } from "./util"
 
 // Interface representing a file reference, with a name and filename property
 export interface FileReference {
@@ -57,43 +55,49 @@ export const eolPosition = 0x3fffffff // End of line position, a large constant
 export const eofPosition: CharPosition = [0x3fffffff, 0] // End of file position, a tuple with a large constant
 
 /**
- * Represents a project containing templates and diagnostics.
- * Provides utility methods to manage templates and diagnose issues.
+ * Groups templates by their directory and determines if JS or TS files are present.
+ * Useful for organizing templates based on their file type.
+ * @returns An array of folder information objects, each containing directory name and file type presence.
  */
-export class Project {
-    readonly templates: PromptScript[] = [] // Array of templates within the project
-    readonly diagnostics: Diagnostic[] = [] // Array of diagnostic records
+export function collectFolders(prj: Project) {
+    const folders: Record<
+        string,
+        { dirname: string; js?: boolean; ts?: boolean }
+    > = {}
+    for (const t of Object.values(prj.scripts).filter(
+        // must have a filename and not propmty
+        (t) => t.filename && !PROMPTY_REGEX.test(t.filename)
+    )) {
+        const dirname = host.path.dirname(t.filename) // Get directory name from the filename
+        const folder = folders[dirname] || (folders[dirname] = { dirname })
+        folder.js = folder.js || GENAI_ANYJS_REGEX.test(t.filename) // Check for presence of JS files
+        folder.ts = folder.ts || GENAI_ANYTS_REGEX.test(t.filename) // Check for presence of TS files
+    }
+    return Object.values(folders) // Return an array of folders with their properties
+}
 
-    _finalizers: (() => void)[] = [] // Array of cleanup functions to be called when project is disposed
+/**
+ * Retrieves a template by its ID.
+ * @param id - The ID of the template to retrieve.
+ * @returns The matching PromptScript or undefined if no match is found.
+ */
+export function resolveScript(prj: Project, id: string) {
+    return prj?.scripts?.find((t) => t.id == id) // Find and return the template with the matching ID
+}
 
-    /**
-     * Groups templates by their directory and determines if JS or TS files are present.
-     * Useful for organizing templates based on their file type.
-     * @returns An array of folder information objects, each containing directory name and file type presence.
-     */
-    folders() {
-        const folders: Record<
-            string,
-            { dirname: string; js?: boolean; ts?: boolean }
-        > = {}
-        for (const t of Object.values(this.templates).filter(
-            // must have a filename and not propmty
-            (t) => t.filename && !PROMPTY_REGEX.test(t.filename)
-        )) {
-            const dirname = host.path.dirname(t.filename) // Get directory name from the filename
-            const folder = folders[dirname] || (folders[dirname] = { dirname })
-            folder.js = folder.js || GENAI_ANYJS_REGEX.test(t.filename) // Check for presence of JS files
-            folder.ts = folder.ts || GENAI_ANYTS_REGEX.test(t.filename) // Check for presence of TS files
-        }
-        return Object.values(folders) // Return an array of folders with their properties
-    }
+export interface ScriptFilterOptions {
+    ids?: string[]
+    groups?: string[]
+    test?: boolean
+}
 
-    /**
-     * Retrieves a template by its ID.
-     * @param id - The ID of the template to retrieve.
-     * @returns The matching PromptScript or undefined if no match is found.
-     */
-    getTemplate(id: string) {
-        return this.templates.find((t) => t.id == id) // Find and return the template with the matching ID
-    }
+export function filterScripts(
+    scripts: PromptScript[],
+    options: ScriptFilterOptions
+) {
+    const { ids, groups, test } = options || {}
+    return scripts
+        .filter((t) => !test || arrayify(t.tests)?.length)
+        .filter((t) => !ids?.length || ids.includes(t.id))
+        .filter((t) => tagFilter(groups, t.group))
 }

packages/core/src/config.ts

@@ -40,7 +40,9 @@ export async function resolveGlobalConfiguration(
                     },
                 })
                 if (validation.schemaError)
-                    throw new Error(validation.schemaError)
+                    throw new Error(
+                        `Configuration error: ` + validation.schemaError
+                    )
                 config = structuralMerge(config, parsed)
             }
         }

packages/core/src/copy.ts

@@ -2,7 +2,6 @@
 // including constructing file paths and handling copy operations,
 // with optional forking functionality.
 
-import { PromptScript } from "./ast" // Import PromptScript type
 import { GENAI_MJS_EXT, GENAI_SRC } from "./constants" // Import constants for file extensions and source directory
 import { host } from "./host" // Import host module for file operations
 import { fileExists, writeText } from "./fs" // Import file system utilities