XML tag using def

docs/src/content/docs/reference/cli/commands.md

@@ -94,10 +94,10 @@
   -o, --out <folder>                  output folder
   -rmo, --remove-out                  remove output folder if it exists
   --cli <string>                      override path to the cli
   -td, --test-delay <string>          delay between tests in seconds
   --cache                             enable LLM result cache
   -v, --verbose                       verbose output
-  -pv, --promptfoo-version [version]  promptfoo version, default is 0.97.0
+  -pv, --promptfoo-version [version]  promptfoo version, default is 0.100.3
   -os, --out-summary <file>           append output summary in file
   -g, --groups <groups...>            groups to include or exclude. Use :!
                                       prefix to exclude

packages/cli/package.json

@@ -67,7 +67,7 @@
     "node": ">=20.0.0"
   },
   "peerDependencies": {
-    "promptfoo": "0.97.0"
+    "promptfoo": "0.100.3"
   },
   "devDependencies": {
     "@types/diff": "^6.0.0",

packages/cli/src/run.ts

@@ -250,24 +250,25 @@ export async function runScript(
     if (GENAI_ANY_REGEX.test(scriptId)) toolFiles.push(scriptId)
 
     for (const arg of files) {
+        if (HTTPS_REGEX.test(arg)) {
+            resolvedFiles.add(arg)
+            continue
+        }
         const stats = await host.statFile(arg)
         if (!stats)
             return fail(`file not found: ${arg}`, FILES_NOT_FOUND_ERROR_CODE)
         if (stats.type !== "file") continue
-        if (HTTPS_REGEX.test(arg)) resolvedFiles.add(arg)
-        else {
-            const ffs = await host.findFiles(arg, {
-                applyGitIgnore: excludeGitIgnore,
-            })
-            if (!ffs?.length) {
-                return fail(
-                    `no files matching ${arg} under ${process.cwd()}`,
-                    FILES_NOT_FOUND_ERROR_CODE
-                )
-            }
-            for (const file of ffs) {
-                resolvedFiles.add(filePathOrUrlToWorkspaceFile(file))
-            }
+        const ffs = await host.findFiles(arg, {
+            applyGitIgnore: excludeGitIgnore,
+        })
+        if (!ffs?.length) {
+            return fail(
+                `no files matching ${arg} under ${process.cwd()}`,
+                FILES_NOT_FOUND_ERROR_CODE
+            )
+        }
+        for (const file of ffs) {
+            resolvedFiles.add(filePathOrUrlToWorkspaceFile(file))
         }
     }
 

packages/core/src/chat.ts

@@ -663,6 +663,7 @@ async function processChatMessage(
                 const { errors, messages: participantMessages } =
                     await renderPromptNode(options.model, node, {
                         flexTokens: options.flexTokens,
+                        fenceFormat: options.fenceFormat,
                         trace,
                     })
                 if (participantMessages?.length) {
@@ -737,7 +738,8 @@ export function mergeGenerationOptions(
             options?.model ??
             runtimeHost.modelAliases.large.model,
         temperature:
-            runOptions?.temperature ?? runtimeHost.modelAliases.large.temperature,
+            runOptions?.temperature ??
+            runtimeHost.modelAliases.large.temperature,
         embeddingsModel:
             runOptions?.embeddingsModel ??
             options?.embeddingsModel ??

packages/core/src/constants.ts

@@ -54,6 +54,7 @@ export const HIGHLIGHT_LENGTH = 4000
 export const SMALL_MODEL_ID = "small"
 export const LARGE_MODEL_ID = "large"
 export const VISION_MODEL_ID = "vision"
+export const DEFAULT_FENCE_FORMAT = "markdown"
 export const DEFAULT_MODEL = "openai:gpt-4o"
 export const DEFAULT_MODEL_CANDIDATES = [
     "azure:gpt-4o",

packages/core/src/encoders.test.ts

@@ -58,7 +58,7 @@ For example, to denote a heading, you add a number sign before it (e.g., # Headi
                 chunkSize: 128,
                 chunkOverlap: 16,
                 model: "gpt-4o",
-                lineNumbers: true
+                lineNumbers: true,
             }
         )
         console.log(chunks)

packages/core/src/expander.ts

@@ -41,8 +41,9 @@ export async function callExpander(
     options: GenerationOptions
 ) {
     assert(!!options.model)
-    const { provider, model } = parseModelIdentifier(r.model ?? options.model)
-    const ctx = await createPromptContext(prj, vars, trace, options, model)
+    const modelId = r.model ?? options.model
+    const { provider } = parseModelIdentifier(modelId)
+    const ctx = await createPromptContext(prj, vars, trace, options, modelId)
 
     let status: GenerationStatus = undefined
     let statusText: string = undefined
@@ -90,8 +91,9 @@ export async function callExpander(
                 fileOutputs: fos,
                 prediction: pred,
                 disposables: mcps,
-            } = await renderPromptNode(model, node, {
+            } = await renderPromptNode(modelId, node, {
                 flexTokens: options.flexTokens,
+                fenceFormat: options.fenceFormat,
                 trace,
             })
             messages = msgs
@@ -213,6 +215,7 @@ export async function expandTemplate(
         normalizeInt(env.vars["flexTokens"]) ??
         normalizeInt(env.vars["flex_tokens"]) ??
         template.flexTokens
+    const fenceFormat = options.fenceFormat ?? template.fenceFormat
     let seed = options.seed ?? normalizeInt(env.vars["seed"]) ?? template.seed
     if (seed !== undefined) seed = seed >> 0
     let logprobs = options.logprobs || template.logprobs
@@ -237,6 +240,7 @@ export async function expandTemplate(
         topP,
         temperature,
         lineNumbers,
+        fenceFormat,
     })
 
     const { status, statusText, messages } = prompt