feat: ✨ add model connection info resolution

packages/cli/src/test.ts

@@ -40,6 +40,10 @@ import {
 } from "../../core/src/server/messages"
 import { generatePromptFooConfiguration } from "../../core/src/test"
 import { delay } from "es-toolkit"
+import {
+    ModelConnectionInfo,
+    resolveModelConnectionInfo,
+} from "../../core/src/models"
 
 /**
  * Parses model specifications from a string and returns a ModelOptions object.
@@ -72,9 +76,12 @@ function parseModelSpec(m: string): ModelOptions {
  * @param script - The PromptScript object for which to determine the provider.
  * @returns The resolved provider or undefined if not found.
  */
-async function resolveTestProvider(script: PromptScript) {
-    const token = await host.getLanguageModelConfiguration(script.model)
-    if (token && token.type === "azure") return token.base
+async function resolveTestProvider(info: ModelConnectionInfo) {
+    switch (info?.type) {
+        case "azure":
+        case "azure_serverless":
+            return info?.base
+    }
     return undefined
 }
 
@@ -147,13 +154,15 @@ export async function runPromptScriptTests(
             ? join(out, `${script.id}.promptfoo.yaml`)
             : script.filename.replace(GENAI_ANY_REGEX, ".promptfoo.yaml")
         logInfo(`  ${fn}`)
+        const { info } = await resolveModelConnectionInfo(script)
+        if (info.error) throw new Error(info.error)
         const testProvider =
-            options?.testProvider || (await resolveTestProvider(scripts[0]))
+            options?.testProvider || (await resolveTestProvider(info))
         const config = generatePromptFooConfiguration(script, {
             out,
             cli,
-            model: options.model,
-            smallModel: options.smallModel,
+            model: info.model,
+            smallModel: info.smallModel,
             models: options.models?.map(parseModelSpec),
             provider: "provider.mjs",
             testProvider,

packages/core/src/config.ts

@@ -0,0 +1,16 @@
+import { PromptScriptRunOptions } from "./server/messages"
+
+/**
+ * Schema for a global configuration file
+ */
+export interface GenAIScriptConfiguration {
+    /**
+     * Path to the .env file
+     */
+    env?: string
+
+    /**
+     * Common run options
+     */
+    run?: PromptScriptRunOptions
+}