Merge remote-tracking branch 'origin/main' into serverless_entra

THIRD_PARTY_LICENSES.md

@@ -1246,7 +1246,7 @@ The following npm packages may be included in this product:
 
  - @types/http-cache-semantics@4.0.4
  - @types/node@16.9.1
- - @types/node@22.7.6
+ - @types/node@22.7.7
  - @types/turndown@5.0.5
  - @types/yauzl@2.10.3
 
@@ -1278,7 +1278,7 @@ MIT License
 
 The following npm package may be included in this product:
 
- - genaiscript-vscode@1.64.6
+ - genaiscript-vscode@1.65.0
 
 This package contains the following license and notice below:
 
@@ -4400,9 +4400,9 @@ The following npm packages may be included in this product:
  - @tokenizer/token@0.3.0
  - [email protected]
  - [email protected]
- - genaiscript-core-internal@1.64.6
- - genaiscript-sample@1.64.6
- - genaiscript@1.64.6
+ - genaiscript-core-internal@1.65.0
+ - genaiscript-sample@1.65.0
+ - genaiscript@1.65.0
  - [email protected]
  - [email protected]
  - [email protected]
@@ -5083,7 +5083,7 @@ SOFTWARE.
 
 The following npm package may be included in this product:
 
- - [email protected].0
+ - [email protected].1
 
 This package contains the following license and notice below:
 

docs/src/content/docs/getting-started/configuration.mdx

@@ -120,7 +120,7 @@ the `.env` file will appear grayed out in Visual Studio Code.
 
 ## OpenAI
 
-This provider, `openai`, is the default provider.
+This provider, `openai`, is the OpenAI chat model provider.
 It uses the `OPENAI_API_...` environment variables.
 
 <Steps>
@@ -437,6 +437,44 @@ model2=key2model3=key3
 "
 ```
 
+## Anthropic
+
+This provider, `anthropic`, is the Anthropic chat model provider.
+It uses the `ANTHROPIC_API_...` environment variables.
+
+<Steps>
+
+<ol>
+
+<li>
+
+Update the `.env` file with the secret key.
+
+```txt title=".env"
+ANTHROPIC_API_KEY=...
+# optional
+ANTHROPIC_API_BASE=...
+```
+
+</li>
+
+<li>
+
+Set the `model` field in `script` to the model you want to use.
+
+```js 'model: "anthropic:claude-2"'
+script({
+    model: "anthropic:claude-2",
+    ...
+})
+```
+
+</li>
+
+</ol>
+
+</Steps>
+
 ## GitHub Copilot Chat Models <a id="github-copilot" href=""></a>
 
 If you have access to **GitHub Copilot Chat in Visual Studio Code**,
@@ -479,6 +517,55 @@ select the best chat model that matches the one you have in your script
 
 The mapping of GenAIScript model names to Visual Studio Models is stored in the settings.
 
+## Anthropic
+
+[Anthropic](https://www.anthropic.com/) is an AI research company that offers powerful language models, including the Claude series.
+
+To use Anthropic models with GenAIScript, follow these steps:
+
+<Steps>
+
+<ol>
+
+<li>
+
+Sign up for an Anthropic account and obtain an API key from their [console](https://console.anthropic.com/).
+
+</li>
+
+<li>
+
+Add your Anthropic API key to the `.env` file:
+
+```txt title=".env"
+ANTHROPIC_API_KEY=sk-ant-api...
+```
+
+</li>
+
+<li>
+
+Find the model that best suits your needs by visiting the [Anthropic model documentation](https://docs.anthropic.com/en/docs/about-claude/models#model-names).
+
+</li>
+
+<li>
+
+Update your script to use the `model` you choose.
+
+```js
+script({
+    ...
+    model: "anthropic:claude-3-5-sonnet-20240620",
+})
+```
+
+</li>
+
+</ol>
+
+</Steps>
+
 ## Local Models
 
 There are many projects that allow you to run models locally on your machine,

docs/src/content/docs/reference/cli/commands.md

@@ -303,6 +303,9 @@ Options:
   -h, --help                   display help for command
 
 Commands:
+  data [options] <file>        Convert CSV, YAML, TOML, INI, XLSX, XML, MD/X
+                               frontmatter or JSON data files into various
+                               formats
   fence <language> <file>      Extracts a code fenced regions of the given type
   pdf <file>                   Parse a PDF into text
   docx <file>                  Parse a DOCX into texts
@@ -312,6 +315,21 @@ Commands:
   tokens [options] <files...>  Count tokens in a set of files
   jsonl2json                   Converts JSONL files to a JSON file
   prompty [options] <file...>  Converts .prompty files to genaiscript
+  jinja2 [options] <file>      Renders Jinj2 or prompty template
+```
+
+### `parse data`
+
+```
+Usage: genaiscript parse data [options] <file>
+
+Convert CSV, YAML, TOML, INI, XLSX, XML, MD/X frontmatter or JSON data files
+into various formats
+
+Options:
+  -f, --format <string>  output format (choices: "json", "json5", "yaml",
+                         "ini", "csv", "md")
+  -h, --help             display help for command
 ```
 
 ### `parse fence`
@@ -407,6 +425,21 @@ Options:
   -h, --help          display help for command
 ```
 
+### `parse jinja2`
+
+```
+Usage: genaiscript parse jinja2 [options] <file>
+
+Renders Jinj2 or prompty template
+
+Arguments:
+  file                   input Jinja2 or prompty template file
+
+Options:
+  --vars <namevalue...>  variables, as name=value passed to the template
+  -h, --help             display help for command
+```
+
 ## `info`
 
 ```

docs/src/content/docs/reference/scripts/parsers.md

@@ -315,3 +315,12 @@ used with `defData`.
 ```js
 const d = parsers.tidyData(rows, { sliceSample: 100, sort: "name" })
 ```
+
+## Command line
+
+Use the [parse](/genaiscript/reference/cli/commands#parse) command from the CLI to try out various parsers.
+
+```sh
+# convert any known data format to JSON
+npx --yes genaiscript parse data mydata.csv
+```

docs/yarn.lock

@@ -1120,9 +1120,9 @@
     "@types/unist" "*"
 
 "@types/node@*", "@types/node@>=20":
-  version "22.7.6"
-  resolved "https://registry.yarnpkg.com/@types/node/-/node-22.7.6.tgz#3ec3e2b071e136cd11093c19128405e1d1f92f33"
-  integrity sha512-/d7Rnj0/ExXDMcioS78/kf1lMzYk4BZV8MZGTBKzTGZ6/406ukkbYlIsZmMPhcR5KlkunDHQLrtAVmSq7r+mSw==
+  version "22.7.7"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-22.7.7.tgz#6cd9541c3dccb4f7e8b141b491443f4a1570e307"
+  integrity sha512-SRxCrrg9CL/y54aiMCG3edPKdprgMVGDXjA3gB8UmmBW5TcXzRUYAh8EWzTnSJFAd1rgImPELza+A3bJ+qxz8Q==
   dependencies:
     undici-types "~6.19.2"
 
@@ -2674,9 +2674,9 @@ http-cache-semantics@^4.1.1:
   integrity sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ==
 
 i18next@^23.11.5:
-  version "23.16.0"
-  resolved "https://registry.yarnpkg.com/i18next/-/i18next-23.16.0.tgz#6127b49a30c19ba5caf0de8aeb3e15df60f7f01e"
-  integrity sha512-Ni3CG6c14teOogY19YNRl+kYaE/Rb59khy0VyHVn4uOZ97E2E/Yziyi6r3C3s9+wacjdLZiq/LLYyx+Cgd+FCw==
+  version "23.16.2"
+  resolved "https://registry.yarnpkg.com/i18next/-/i18next-23.16.2.tgz#ef6868ef6ac7bcaee4f5fe35132021d44e8d1063"
+  integrity sha512-dFyxwLXxEQK32f6tITBMaRht25mZPJhQ0WbC0p3bO2mWBal9lABTMqSka5k+GLSRWLzeJBKDpH7BeIA9TZI7Jg==
   dependencies:
     "@babel/runtime" "^7.23.2"
 

package.json

@@ -69,7 +69,8 @@
         "genai:docify": "node packages/cli/built/genaiscript.cjs run docify",
         "gcm": "node packages/cli/built/genaiscript.cjs run gcm",
         "genai": "node packages/cli/built/genaiscript.cjs run",
-        "upgrade:deps": "zx scripts/upgrade-deps.mjs"
+        "upgrade:deps": "zx scripts/upgrade-deps.mjs",
+        "cli": "node packages/cli/built/genaiscript.cjs"
     },
     "release-it": {
         "github": {

packages/cli/src/cli.ts

@@ -4,7 +4,7 @@
  */
 
 import { NodeHost } from "./nodehost" // Handles node environment setup
-import { program } from "commander" // Command-line argument parsing library
+import { Option, program } from "commander" // Command-line argument parsing library
 import { error, isQuiet, setConsoleColors, setQuiet } from "./log" // Logging utilities
 import { startServer } from "./server" // Function to start server
 import { NODE_MIN_VERSION, PROMPTFOO_VERSION } from "./version" // Version constants
@@ -13,9 +13,11 @@ import { retrievalFuzz, retrievalSearch } from "./retrieval" // Retrieval functi
 import { helpAll } from "./help" // Display help for all commands
 import {
     jsonl2json,
+    parseAnyToJSON,
     parseDOCX,
     parseFence,
     parseHTMLToText,
+    parseJinja2,
     parsePDF,
     parseTokens,
     prompty2genaiscript,
@@ -293,26 +295,38 @@ export async function cli() {
         .command("parse")
         .alias("parsers")
         .description("Parse various outputs")
+    const parserData = parser
+        .command("data <file>")
+        .description(
+            "Convert CSV, YAML, TOML, INI, XLSX, XML, MD/X frontmatter or JSON data files into various formats"
+        )
+        .action(parseAnyToJSON)
+    parserData.addOption(
+        new Option("-f, --format <string>", "output format").choices([
+            "json",
+            "json5",
+            "yaml",
+            "ini",
+            "csv",
+            "md",
+        ])
+    )
     parser
         .command("fence <language> <file>")
         .description("Extracts a code fenced regions of the given type")
         .action(parseFence) // Action to parse fenced code regions
-
     parser
         .command("pdf <file>")
         .description("Parse a PDF into text")
         .action(parsePDF) // Action to parse PDF files
-
     parser
         .command("docx <file>")
         .description("Parse a DOCX into texts")
         .action(parseDOCX) // Action to parse DOCX files
-
     parser
         .command("html-to-text <file>")
         .description("Parse an HTML file into text")
         .action(parseHTMLToText) // Action to parse HTML files
-
     parser
         .command("code")
         .description("Parse code using tree sitter and executes a query")
@@ -334,6 +348,15 @@ export async function cli() {
         .argument("<file...>", "input JSONL files")
         .option("-o, --out <string>", "output folder")
         .action(prompty2genaiscript) // Action to convert prompty files
+    parser
+        .command("jinja2")
+        .description("Renders Jinj2 or prompty template")
+        .argument("<file>", "input Jinja2 or prompty template file")
+        .option(
+            "--vars <namevalue...>",
+            "variables, as name=value passed to the template"
+        )
+        .action(parseJinja2)
 
     // Define 'info' command group for utility information tasks
     const info = program.command("info").description("Utility tasks")