handle multiple scripts

microsoft · Apr 29, 2024 · bb91398 · bb91398
1 parent e87f40f
commit bb91398
Show file tree

Hide file tree

Showing 6 changed files with 66 additions and 38 deletions.
diff --git a/packages/cli/src/server.ts b/packages/cli/src/server.ts
@@ -10,7 +10,7 @@ import {
     ServerResponse,
     serializeError,
 } from "genaiscript-core"
-import { scriptsTest } from "./test"
+import { runTests } from "./test"
 
 export async function startServer(options: { port: string }) {
     const port = parseInt(options.port) || SERVER_PORT
@@ -77,14 +77,13 @@ export async function startServer(options: { port: string }) {
                         break
                     }
                     case "tests.run": {
-                        console.log(`tests: run ${data.script || "*"}`)
-                        await scriptsTest(data.script, {
+                        console.log(
+                            `tests: run ${data.scripts?.join(", ") || "*"}`
+                        )
+                        response = await runTests(data.scripts, {
                             ...(data.options || {}),
-                            cache: true
+                            cache: true,
                         })
-                        response = {
-                            ok: true,
-                        }
                         break
                     }
                     default:

diff --git a/packages/cli/src/test.ts b/packages/cli/src/test.ts
@@ -3,9 +3,11 @@ import { buildProject } from "./build"
 import {
     EXEC_MAX_BUFFER,
     GENAISCRIPT_FOLDER,
+    ResponseStatus,
     TestRunOptions,
     YAMLStringify,
     arrayify,
+    host,
     logVerbose,
     normalizeFloat,
     parseKeyValuePairs,
@@ -27,28 +29,38 @@ function parseModelSpec(m: string): ModelOptions {
     else return { model: m }
 }
 
-export async function scriptsTest(
-    id: string,
+async function resolveTestProvider(script: PromptScript) {
+    const token = await host.getSecretToken(script)
+    if (token && token.type === "azure") return token.base
+    return undefined
+}
+
+export async function runTests(
+    ids: string[],
     options: TestRunOptions & {
         out?: string
         cli?: string
         removeOut?: boolean
-        view?: boolean
         cache?: boolean
         verbose?: boolean
         write?: boolean
     }
-) {
+): Promise<ResponseStatus> {
     const prj = await buildProject()
     const scripts = prj.templates
         .filter((t) => arrayify(t.tests)?.length)
-        .filter((t) => !id || t.id === id)
-    if (!scripts.length) throw new Error(`no script with tests found`)
+        .filter((t) => !ids?.length || ids.includes(t.id))
+    if (!scripts.length)
+        return {
+            ok: false,
+            status: 404,
+        }
 
     const cli = options.cli || __filename
     const out = options.out || join(GENAISCRIPT_FOLDER, "tests")
     const provider = join(out, "provider.mjs")
-    const testProvider = options?.testProvider
+    const testProvider =
+        options?.testProvider || (await resolveTestProvider(scripts[0]))
     const models = options?.models
     logVerbose(`writing tests to ${out}`)
 
@@ -96,11 +108,26 @@ export async function scriptsTest(
     exec.pipeStdout(process.stdout)
     exec.pipeStderr(process.stdout)
     const res = await exec
+    return { ok: res.exitCode === 0, status: res.exitCode }
+}
 
+export async function scriptsTest(
+    id: string,
+    options: TestRunOptions & {
+        out?: string
+        cli?: string
+        removeOut?: boolean
+        view?: boolean
+        cache?: boolean
+        verbose?: boolean
+        write?: boolean
+    }
+) {
+    const { status } = await runTests(id, options)
     if (options.view)
         await execa("npx", ["--yes", "promptfoo@latest", "view", "-y"], {
             cleanup: true,
             maxBuffer: EXEC_MAX_BUFFER,
         })
-    else process.exit(res.exitCode)
+    else process.exit(status)
 }
diff --git a/packages/core/src/clihelp.ts b/packages/core/src/clihelp.ts
@@ -56,7 +56,7 @@ ${generateCliArguments(template, fragment, options, "run")}
         trace.details(
             "🧪 testing",
             `
-- test configuration ([promptfoo](https://www.promptfoo.dev/))
+- [promptfoo](https://www.promptfoo.dev/) configuration
 
 \`\`\`yaml
 ${YAMLStringify(generatePromptFooConfiguration(template, { models: [options] }))}

diff --git a/packages/core/src/server/client.ts b/packages/core/src/server/client.ts
@@ -154,7 +154,7 @@ export class WebSocketClient implements RetrievalService, ParseService {
     async runTest(script: PromptScript, options: TestRunOptions) {
         const res = await this.queue<TestRunMessage>({
             type: "tests.run",
-            script: script?.id,
+            scripts: script?.id ? [script?.id] : undefined,
             options,
         })
         return res.response

diff --git a/packages/core/src/server/messages.ts b/packages/core/src/server/messages.ts
@@ -53,7 +53,7 @@ export interface TestRunOptions {
 
 export interface TestRunMessage extends RequestMessage {
     type: "tests.run"
-    script?: string
+    scripts?: string[]
     options?: TestRunOptions
 }
 

diff --git a/packages/core/src/test.ts b/packages/core/src/test.ts
@@ -36,26 +36,28 @@ export function generatePromptFooConfiguration(
                 cli,
             },
         })),
-        defaultTest: {
-            options: {
-                provider: testProvider
-                    ? {
-                          text: {
-                              id: "azureopenai:chat:gpt-4",
-                              config: {
-                                  apiHost: "tnrllmproxy.azurewebsites.net",
-                              },
-                          },
-                          embedding: {
-                              id: "azureopenai:embeddings:text-embedding-ada-002",
-                              config: {
-                                  apiHost: "tnrllmproxy.azurewebsites.net",
-                              },
-                          },
-                      }
-                    : undefined,
-            },
-        },
+        defaultTest: testProvider
+            ? {
+                  options: {
+                      provider: testProvider
+                          ? {
+                                text: {
+                                    id: "azureopenai:chat:gpt-4",
+                                    config: {
+                                        apiHost: testProvider,
+                                    },
+                                },
+                                embedding: {
+                                    id: "azureopenai:embeddings:text-embedding-ada-002",
+                                    config: {
+                                        apiHost: testProvider,
+                                    },
+                                },
+                            }
+                          : undefined,
+                  },
+              }
+            : undefined,
         tests: arrayify(tests).map(
             ({
                 description,