XML tag using def

.gitignore

@@ -28,3 +28,6 @@ trapi.md
 genaiscript.config.yaml
 packages/core/*.temp.*
 *.vsix
+packages/sample/test.txt
+packages/sample/poems/*.txt
+packages/sample/src/rag/markdown.md.txt

docs/genaisrc/frontmatter.genai.mjs

@@ -43,10 +43,14 @@ For each FILE, re-generate the front matter content as the new file content.
 `
 
 // merge logic to integrate generated frontmatter fields
-defFileMerge(function frontmatter(fn, label, before, generated) {
+defFileOutput("**/*.{md,mdx}", "Updated markdown files")
+defFileMerge((fn, label, before, generated) => {
     if (!/\.mdx?$/i.test(fn)) return undefined
     const frontmatter = MD.frontmatter(generated)
-    if (!frontmatter) return before
+    if (!frontmatter) {
+        console.log(`invalid syntax for generated frontmatter`)
+        return before
+    }
 
     const { title, description, keywords, tags } = frontmatter
     const updated = MD.updateFrontmatter(before, {

docs/src/content/docs/reference/cli/commands.md

@@ -52,6 +52,7 @@ Options:
   --cache                                    enable LLM result cache
   -cn, --cache-name <name>                   custom cache file name
   -cs, --csv-separator <string>              csv separator (default: "\t")
+  -ff, --fence-format <string>               fence format (choices: "xml", "markdown", "none")
   -ae, --apply-edits                         apply file edits
   --vars <namevalue...>                      variables, as name=value, stored in env.vars. Use environment variables GENAISCRIPT_VAR_name=value to pass variable through the environment
   -rr, --run-retry <number>                  number of retries for the entire run
@@ -97,7 +98,7 @@ Options:
   -td, --test-delay <string>          delay between tests in seconds
   --cache                             enable LLM result cache
   -v, --verbose                       verbose output
-  -pv, --promptfoo-version [version]  promptfoo version, default is 0.97.0
+  -pv, --promptfoo-version [version]  promptfoo version, default is 0.100.3
   -os, --out-summary <file>           append output summary in file
   -g, --groups <groups...>            groups to include or exclude. Use :!
                                       prefix to exclude

docs/src/content/docs/reference/scripts/context.md

@@ -18,7 +18,7 @@ The `env` global object contains properties that provide information about the s
 The `env.files` array contains all files within the execution context. The context is defined implicitly
 by the user based on:
 
--   `script` `files` option
+- `script` `files` option
 
 ```js
 script({
@@ -34,26 +34,26 @@ script({
 })
 ```
 
--   the UI location to start the tool
+- the UI location to start the tool
 
--   [CLI](/genaiscript/reference/cli) files arguments.
+- [CLI](/genaiscript/reference/cli) files arguments.
 
 The files are stored in `env.files` which can be injected in the prompt.
 
--   using `def`
+- using `def`
 
 ```js
 def("FILE", env.files)
 ```
 
--   filtered,
+- filtered,
 
 ```js
 def("DOCS", env.files, { endsWith: ".md" })
 def("CODE", env.files, { endsWith: ".py" })
 ```
 
--   directly in a `$` call
+- directly in a `$` call
 
 ```js
 $`Summarize ${env.files}.
@@ -81,21 +81,29 @@ Read more about [variables](/genaiscript/reference/scripts/variables).
 ## Definition (`def`)
 
 The `def("FILE", file)` function is a shorthand for generating a fenced variable output.
-The "meta-variable" (`FILE` in this example) name should be all uppercase (but can include
 
 ```js "def"
 def("FILE", file)
 ```
 
-approximately equivalent to:
+It renders approximately to
 
-````js
-$`FILE ${file.filename}:
-```
-${env.file.content}
+````markdown
+FILE:
+
+```file="filename"
+file content
 ```
 ````
 
+or if the model support XML tags (see [fence formats](/genaiscript/reference/scripts/fence-formats)):
+
+```markdown
+<FILE file="filename">
+file content
+</FILE>
+```
+
 The `def` function can also be used with an array of files, such as `env.files`.
 
 ```js "env.files"
@@ -126,13 +134,13 @@ $`Summarize ${f}.`
 
 Since a script may be executed on a full folder, it is often useful to filter the files based on
 
--   their extension
+- their extension
 
 ```js "endsWith: '.md'"
 def("FILE", env.files, { endsWith: ".md" })
 ```
 
--   or using a [glob](<https://en.wikipedia.org/wiki/Glob_(programming)>):
+- or using a [glob](<https://en.wikipedia.org/wiki/Glob_(programming)>):
 
 ```js "glob: '**/*.{md,mdx}'"
 def("FILE", files, { glob: "**/*.{md,mdx}" })
@@ -172,19 +180,19 @@ def("FILE", env.files, { maxTokens: 100 })
 The `def` function treats data files such as [CSV](/genaiscript/reference/scripts/csv) and [XLSX](/genaiscript/reference/scripts/xlsx) specially. It will automatically convert the data into a
 markdown table format to improve tokenization.
 
--   `sliceHead`, keep the top N rows
+- `sliceHead`, keep the top N rows
 
 ```js "sliceHead: 100"
 def("FILE", env.files, { sliceHead: 100 })
 ```
 
--   `sliceTail`, keep the last N rows
+- `sliceTail`, keep the last N rows
 
 ```js "sliceTail: 100"
 def("FILE", env.files, { sliceTail: 100 })
 ```
 
--   `sliceSample`, keep a random sample of N rows
+- `sliceSample`, keep a random sample of N rows
 
 ```js "sliceSample: 100"
 def("FILE", env.files, { sliceSample: 100 })
@@ -238,11 +246,11 @@ defData("DATA", csv, { format: "yaml" })
 
 The `defData` function also supports functions to slice the input rows and columns.
 
--   `headers`, list of column names to include
--   `sliceHead`, number of rows to include from the beginning
--   `sliceTail`, number of rows to include from the end
--   `sliceSample`, number of rows to pick at random
--   `distinct`, list of column names to deduplicate the data based on
+- `headers`, list of column names to include
+- `sliceHead`, number of rows to include from the beginning
+- `sliceTail`, number of rows to include from the end
+- `sliceSample`, number of rows to pick at random
+- `distinct`, list of column names to deduplicate the data based on
 
 ```js
 defData("DATA", data, {

docs/src/content/docs/reference/scripts/fence-formats.md

@@ -0,0 +1,82 @@
+---
+title: Fence Formats
+sidebar:
+    order: 90
+description: Explore various fence formats supported by GenAIScript for optimal
+    LLM input text formatting.
+keywords: fence format, LLM input, markdown, xml, GenAIScript
+---
+
+GenAIScript supports various types of "fence" formats when rendering [def](/genaiscript/reference/scripts/content) function, since LLMs may behave differently depending on the format of the input text.
+**As of 1.82.0, the default format is to use XML tags.**
+
+- [Anthropic](https://docs.anthropic.com/en/docs/build-with-claude/prompt-engineering/use-xml-tags)
+- [OpenAI](https://platform.openai.com/docs/guides/prompt-engineering#tactic-use-delimiters-to-clearly-indicate-distinct-parts-of-the-input)
+- [Google](https://cloud.google.com/vertex-ai/generative-ai/docs/learn/prompts/structure-prompts)
+
+The following `def` call will generate a fenced region with different syntax:
+
+- `xml`
+
+```js
+def("TEXT", ":)", { fenceFormat: "xml" })
+```
+
+```markdown
+<TEXT>
+:)
+</TEXT>
+```
+
+- `markdown`
+
+```js
+def("TEXT", ":)", { fenceFormat: "markdown" })
+```
+
+```markdown
+TEXT:
+\`\`\`
+:)
+\`\`\`
+```
+
+- `none`
+
+```js
+def("TEXT", ":)", { fenceFormat: "none" })
+```
+
+```text
+TEXT:
+:)
+```
+
+## Referencing a def
+
+If you are using the `xml` format, it is advised to use `<NAME>` when referencing the `def` variable, or use the returned value as the name.
+
+```js
+const textName = def("TEXT", ":)", { fenceFormat: "xml" })
+$`Summarize ${textName}` // Summarize <TEXT>
+```
+
+## Configuriation
+
+GenAIScript will automatically pick a format based on the model. However, you can override the format at the script level.
+
+```js
+script({ fenceFormat: "xml" })
+```
+
+or at the `def` level:
+
+```js
+def("TEXT", ":)", { fenceFormat: "xml" })
+```
+
+or through the `--fence-format` flag on the cli:
+
+```sh
+npx --yes genaiscript run ... --fence-format xml
+```

docs/src/content/docs/reference/scripts/images.md

@@ -20,7 +20,7 @@ Read more about [OpenAI Vision](https://platform.openai.com/docs/guides/vision/l
 
 Public URLs (that do not require authentication) will be passed directly to OpenAI.
 
-```js
+```js wrap
 defImages(
     "https://github.com/microsoft/genaiscript/blob/main/docs/public/images/logo.png?raw=true"
 )
@@ -35,7 +35,7 @@ and [Blob](https://developer.mozilla.org/en-US/docs/Web/API/Blob).
 
 This example takes a screenshot of bing.com and adds it to the images.
 
-```js
+```js wrap
 const page = await host.browse("https://bing.com")
 const screenshot = await page.screenshot() // returns a node.js Buffer
 defImages(screenshot)
@@ -54,23 +54,23 @@ defImages(img, { detail: "low" })
 
 You can crop a region of interest from the image.
 
-```js "crop: { x: 0, y: 0, w: 512, h: 512 }"
+```js "crop: { x: 0, y: 0, w: 512, h: 512 }" wrap
 defImages(img, { crop: { x: 0, y: 0, w: 512, h: 512 } })
 ```
 
 ## Auto crop
 
 You can also automatically remove uniform color on the edges of the image.
 
-```js "autoCrop"
+```js "autoCrop" wrap
 defImages(img, { autoCrop: true })
 ```
 
 ## Greyscale
 
 You can convert the image to greyscale.
 
-```js "greyscale"
+```js "greyscale" wrap
 defImages(img, { greyscale: true })
 ```
 
@@ -94,7 +94,7 @@ defImages(img, { scale: 0.5 })
 
 You can flip the image.
 
-```js "flip: { horizontal: true; vertical: true }"
+```js "flip: { horizontal: true; vertical: true }" wrap
 defImages(img, { flip: { horizontal: true; vertical: true } })
 ```
 

docs/src/content/docs/reference/scripts/structured-output.md

@@ -6,7 +6,7 @@ description: Utilize structured output in GenAIScript to generate JSON data with
 keywords: structured output, JSON schema, GenAIScript, data validation, reliable data
 ---
 
-Structured output is a feature that allows you to generate structured data in JSON format with a [JSON schema](/genaiscript/reference/scripts/schemas). This is more strict than [JSON mode](/genaiscript/reference/scripts/json-mode) and is supported by `gpt-4o-mini`, `gpt-4o-2024-08-06` and [later models](https://platform.openai.com/docs/guides/structured-outputs/structured-outputs-vs-json-mode).
+Structured output is a feature that allows you to generate structured data in JSON format with a [JSON schema](/genaiscript/reference/scripts/schemas). This is more strict than [JSON mode](/genaiscript/reference/scripts/json-mode).
 
 To enable this mode, set `responseType` to `json_schema`.
 
@@ -26,8 +26,15 @@ script({
 
 Note that there are [several restrictions](https://platform.openai.com/docs/guides/structured-outputs/how-to-use) on the schema features supported by this mode.
 
--   `additionalProperties: true` is not supported.
--   all optional fields (e.g. not in `required`) will be returned and might be `null`
+- `additionalProperties: true` is not supported.
+- all optional fields (e.g. not in `required`) will be returned and might be `null`
+
+## Support
+
+This is a list of models that support structured output. This is a changing field and the list might not be up-to-date. The best way is to try it out.
+
+- OpenAI: supported by `gpt-4o-mini`, `gpt-4o-2024-08-06` and [later models](https://platform.openai.com/docs/guides/structured-outputs/structured-outputs-vs-json-mode).
+- Ollama: supported by 0.5.0 and higher
 
 ## Choices
 

docs/src/content/docs/reference/scripts/system.mdx

@@ -662,20 +662,30 @@ system({
 const folder = env.vars["outputFolder"] || "."
 $`## FILE file format
 
-When generating, saving or updating files you should use the FILE file syntax preferably:`
+When generating, saving or updating files you should use the FILE file syntax preferably:
 
-def(`File ${folder}/file1.ts`, `What goes in\n${folder}/file1.ts.`, {
-    language: "typescript",
-})
-def(`File ${folder}/file1.js`, `What goes in\n${folder}/file1.js.`, {
-    language: "javascript",
-})
-def(`File ${folder}/file1.py`, `What goes in\n${folder}/file1.py.`, {
-    language: "python",
-})
-def(`File /path_to_file/file2.md`, `What goes in\n/path_to_file/file2.md.`, {
-    language: "markdown",
-})
+File ${folder}/file1.ts:
+\`\`\`typescript
+What goes in\n${folder}/file1.ts.
+\`\`\`
+
+File ${folder}/file1.js:
+\`\`\`javascript
+What goes in\n${folder}/file1.js.
+\`\`\`
+
+
+File ${folder}/file1.py: 
+\`\`\`python
+What goes in\n${folder}/file1.py.
+\`\`\`
+
+
+File /path/to/file/file2.md: 
+\`\`\`markdown
+What goes in\n/path/to/file/file2.md.
+\`\`\`
+`
 
 $`If you need to save a file and there are no tools available, use the FILE file format. The output of the LLM will parsed 
 and saved. It is important to use the proper syntax.`

packages/cli/package.json

@@ -67,7 +67,7 @@
     "node": ">=20.0.0"
   },
   "peerDependencies": {
-    "promptfoo": "0.97.0"
+    "promptfoo": "0.100.3"
   },
   "devDependencies": {
     "@types/diff": "^6.0.0",