feat: 🎨 add Jinja2 template parsing feature

docs/src/content/docs/reference/cli/commands.md

@@ -303,8 +303,8 @@ Options:
   -h, --help                   display help for command
 
 Commands:
-  data [options] <file>        Convert CSV, YAML, TOML, INI, XLSX or JSON files
-                               into various formats
+  data [options] <file>        Convert CSV, YAML, TOML, INI, XLSX or JSON data
+                               files into various formats
   fence <language> <file>      Extracts a code fenced regions of the given type
   pdf <file>                   Parse a PDF into text
   docx <file>                  Parse a DOCX into texts
@@ -314,14 +314,15 @@ Commands:
   tokens [options] <files...>  Count tokens in a set of files
   jsonl2json                   Converts JSONL files to a JSON file
   prompty [options] <file...>  Converts .prompty files to genaiscript
+  jinja2 [options] <file>      Renders Jinj2 or prompty template
 ```
 
 ### `parse data`
 
 ```
 Usage: genaiscript parse data [options] <file>
 
-Convert CSV, YAML, TOML, INI, XLSX or JSON files into various formats
+Convert CSV, YAML, TOML, INI, XLSX or JSON data files into various formats
 
 Options:
   -f, --format <string>  output format (choices: "json", "json5", "yaml",
@@ -422,6 +423,21 @@ Options:
   -h, --help          display help for command
 ```
 
+### `parse jinja2`
+
+```
+Usage: genaiscript parse jinja2 [options] <file>
+
+Renders Jinj2 or prompty template
+
+Arguments:
+  file                   input Jinja2 or prompty template file
+
+Options:
+  --vars <namevalue...>  variables, as name=value passed to the template
+  -h, --help             display help for command
+```
+
 ## `info`
 
 ```

packages/cli/src/cli.ts

@@ -17,6 +17,7 @@ import {
     parseDOCX,
     parseFence,
     parseHTMLToText,
+    parseJinja2,
     parsePDF,
     parseTokens,
     prompty2genaiscript,
@@ -347,6 +348,15 @@ export async function cli() {
         .argument("<file...>", "input JSONL files")
         .option("-o, --out <string>", "output folder")
         .action(prompty2genaiscript) // Action to convert prompty files
+    parser
+        .command("jinja2")
+        .description("Renders Jinj2 or prompty template")
+        .argument("<file>", "input Jinja2 or prompty template file")
+        .option(
+            "--vars <namevalue...>",
+            "variables, as name=value passed to the template"
+        )
+        .action(parseJinja2)
 
     // Define 'info' command group for utility information tasks
     const info = program.command("info").description("Utility tasks")

packages/cli/src/parse.ts

@@ -19,6 +19,8 @@ import {
     DEFAULT_MODEL,
     INI_REGEX,
     JSON5_REGEX,
+    MD_REGEX,
+    PROMPTY_REGEX,
     TOML_REGEX,
     XLSX_REGEX,
     YAML_REGEX,
@@ -30,6 +32,9 @@ import { INIParse, INIStringify } from "../../core/src/ini"
 import { TOMLParse } from "../../core/src/toml"
 import { JSON5parse, JSON5Stringify } from "../../core/src/json5"
 import { XLSXParse } from "../../core/src/xlsx"
+import { jinjaRender } from "../../core/src/jinja"
+import { splitMarkdown } from "../../core/src/frontmatter"
+import { parseOptionsVars } from "./vars"
 
 /**
  * This module provides various parsing utilities for different file types such
@@ -83,6 +88,21 @@ export async function parseHTMLToText(file: string) {
     console.log(text)
 }
 
+export async function parseJinja2(
+    file: string,
+    options: {
+        vars: string[]
+    }
+) {
+    let src = await readFile(file, { encoding: "utf-8" })
+    if (PROMPTY_REGEX.test(file)) src = promptyParse(src).content
+    else if (MD_REGEX.test(file)) src = splitMarkdown(src).content
+
+    const vars = parseOptionsVars(options.vars, process.env)
+    const res = jinjaRender(src, vars)
+    console.log(res)
+}
+
 export async function parseAnyToJSON(
     file: string,
     options: { format: string }

packages/cli/src/run.ts

@@ -80,23 +80,7 @@ import { delay } from "es-toolkit"
 import { GenerationStats } from "../../core/src/usage"
 import { traceAgentMemory } from "../../core/src/agent"
 import { appendFile } from "node:fs/promises"
-
-function parseVars(
-    vars: string[],
-    env: Record<string, string>
-): Record<string, string> {
-    const vals =
-        vars?.reduce((acc, v) => ({ ...acc, ...parseKeyValuePair(v) }), {}) ??
-        {}
-    const envVals = Object.keys(env)
-        .filter((k) => CLI_ENV_VAR_RX.test(k))
-        .map((k) => ({
-            [k.replace(CLI_ENV_VAR_RX, "").toLocaleLowerCase()]: env[k],
-        }))
-        .reduce((acc, v) => ({ ...acc, ...v }), {})
-
-    return { ...vals, ...envVals }
-}
+import { parseOptionsVars } from "./vars"
 
 async function setupTraceWriting(trace: MarkdownTrace, filename: string) {
     logVerbose(`trace: ${filename}`)
@@ -271,7 +255,7 @@ export async function runScript(
     const fragment: Fragment = {
         files: Array.from(resolvedFiles),
     }
-    const vars = parseVars(options.vars, process.env)
+    const vars = parseOptionsVars(options.vars, process.env)
     const stats = new GenerationStats("")
     try {
         if (options.label) trace.heading(2, options.label)

packages/cli/src/vars.ts

@@ -0,0 +1,19 @@
+import { CLI_ENV_VAR_RX } from "../../core/src/constants"
+import { parseKeyValuePair } from "../../core/src/fence"
+
+export function parseOptionsVars(
+    vars: string[],
+    env: Record<string, string>
+): Record<string, string> {
+    const vals =
+        vars?.reduce((acc, v) => ({ ...acc, ...parseKeyValuePair(v) }), {}) ??
+        {}
+    const envVals = Object.keys(env)
+        .filter((k) => CLI_ENV_VAR_RX.test(k))
+        .map((k) => ({
+            [k.replace(CLI_ENV_VAR_RX, "").toLocaleLowerCase()]: env[k],
+        }))
+        .reduce((acc, v) => ({ ...acc, ...v }), {})
+
+    return { ...vals, ...envVals }
+}

packages/core/src/constants.ts

@@ -34,6 +34,7 @@ export const TOML_REGEX = /\.toml$/i
 export const XLSX_REGEX = /\.xlsx$/i
 export const DOCX_REGEX = /\.docx$/i
 export const PDF_REGEX = /\.pdf$/i
+export const MD_REGEX = /\.md$/i
 export const MDX_REGEX = /\.mdx$/i
 export const MJS_REGEX = /\.mjs$/i
 export const JS_REGEX = /\.js$/i