Add support for converting .prompty files to genaiscript in CLI and c…

…ore modules

docs/src/content/docs/reference/cli/commands.md

@@ -2,10 +2,11 @@
 title: Commands
 description: List of all CLI commands
 sidebar:
-  order: 100
+    order: 100
 ---
 
 <!-- autogenerated, do not edit -->
+
 A full list of the CLI command and its respective help text.
 
 ## `run`
@@ -309,6 +310,7 @@ Commands:
                                query
   tokens [options] <files...>  Count tokens in a set of files
   jsonl2json                   Converts JSONL files to a JSON file
+  prompty <file...>            Converts .prompty files to genaiscript
 ```
 
 ### `parse fence`
@@ -389,6 +391,20 @@ Options:
   -h, --help  display help for command
 ```
 
+### `parse prompty`
+
+```
+Usage: genaiscript parse prompty [options] <file...>
+
+Converts .prompty files to genaiscript
+
+Arguments:
+  file        input JSONL files
+
+Options:
+  -h, --help  display help for command
+```
+
 ## `workspace`
 
 ```

packages/cli/src/cli.ts

@@ -13,6 +13,7 @@ import {
     parseHTMLToText,
     parsePDF,
     parseTokens,
+    prompty2genaiscript,
 } from "./parse"
 import { compileScript, createScript, fixScripts, listScripts } from "./scripts"
 import { codeQuery } from "./codequery"
@@ -299,6 +300,11 @@ export async function cli() {
         .command("jsonl2json", "Converts JSONL files to a JSON file")
         .argument("<file...>", "input JSONL files")
         .action(jsonl2json)
+    parser
+        .command("prompty")
+        .description("Converts .prompty files to genaiscript")
+        .argument("<file...>", "input JSONL files")
+        .action(prompty2genaiscript)
 
     const workspace = program
         .command("workspace")

packages/cli/src/parse.ts

@@ -10,6 +10,7 @@ import { estimateTokens } from "../../core/src/tokens"
 import { YAMLStringify } from "../../core/src/yaml"
 import { resolveTokenEncoder } from "../../core/src/encoders"
 import { DEFAULT_MODEL } from "../../core/src/constants"
+import { promptyParse, promptyToGenAIScript } from "../../core/src/prompty"
 
 export async function parseFence(language: string, file: string) {
     const res = await parsePdf(file)
@@ -69,3 +70,15 @@ export async function parseTokens(
     }
     console.log(text)
 }
+
+export async function prompty2genaiscript(files: string[]) {
+    const fs = await expandFiles(files)
+    for (const f of fs) {
+        console.log(f)
+        const gf = replaceExt(f, ".genai.mts")
+        const content = await readText(f)
+        const doc = promptyParse(content)
+        const script = promptyToGenAIScript(doc)
+        await writeText(gf, script)
+    }
+}

packages/core/src/json5.ts

@@ -1,5 +1,5 @@
 /* eslint-disable curly */
-import { parse } from "json5"
+import { parse, stringify } from "json5"
 import { jsonrepair } from "jsonrepair"
 
 export function isJSONObjectOrArray(text: string) {
@@ -61,3 +61,5 @@ export function JSONLLMTryParse(s: string): any {
         s = s.replace(startRx, "").replace(endRx, "")
     return JSON5TryParse(s)
 }
+
+export const JSON5Stringify = stringify

packages/core/src/prompty.ts

@@ -1,6 +1,9 @@
 import { ChatCompletionMessageParam } from "./chattypes"
 import { splitMarkdown } from "./frontmatter"
 import { YAMLTryParse } from "./yaml"
+import { dedent } from "./indent"
+import { deleteUndefinedValues } from "./util"
+import { JSON5Stringify } from "./json5"
 
 export interface PromptyFrontmatter {
     name?: string
@@ -9,13 +12,20 @@ export interface PromptyFrontmatter {
     authors?: string[]
     tags?: string[]
     sample?: Record<string, any>
+    inputs?: Record<
+        string,
+        JSONSchemaNumber | JSONSchemaBoolean | JSONSchemaString
+    >
+    outputs?: JSONSchemaObject
 }
 
-export function promptyParse(text: string): {
+export interface PromptyDocument {
     frontmatter: PromptyFrontmatter
     content: string
     messages: ChatCompletionMessageParam[]
-} {
+}
+
+export function promptyParse(text: string): PromptyDocument {
     const { frontmatter = "", content = "" } = splitMarkdown(text)
     const fm = YAMLTryParse(frontmatter) ?? {}
     // todo: validate frontmatter?
@@ -31,7 +41,7 @@ export function promptyParse(text: string): {
         if (role && chunk.length && chunk.some((l) => !!l)) {
             messages.push({
                 role,
-                content: chunk.join("\n"),
+                content: chunk.join("\n").trim(),
             })
         }
     }
@@ -50,3 +60,38 @@ export function promptyParse(text: string): {
     pushMessage()
     return { frontmatter: fm, content, messages }
 }
+
+export function promptyToGenAIScript(doc: PromptyDocument) {
+    const { frontmatter, messages } = doc
+    const { name, description, tags, sample, inputs, outputs } = frontmatter
+    const parameters = inputs ? structuredClone(inputs) : undefined
+    if (parameters && sample)
+        for (const p in sample) {
+            const s = sample[p]
+            if (s !== undefined) parameters[p].default = s
+        }
+    const meta = deleteUndefinedValues(<PromptArgs>{
+        title: name,
+        description,
+        tags,
+        parameters,
+        responseType: outputs ? "json_object" : undefined,
+        responseSchema: outputs,
+    })
+
+    let src = ``
+    if (Object.keys(meta).length) {
+        src += `script(${JSON5Stringify(meta, null, 2)})\n\n`
+    }
+    src += messages
+        .map((m) => {
+            const text = String(m.content).replace(
+                /\{\{([^\}]+)\}\}/g,
+                (m, name) => "${env.vars." + name + "}"
+            )
+            return `$\`${text}\``
+        })
+        .join("\n")
+
+    return src
+}

packages/core/src/util.ts

@@ -49,6 +49,11 @@ export function parseBoolean(s: string) {
           : undefined
 }
 
+export function deleteUndefinedValues<T extends Record<string, any>>(o: T): T {
+    for (const k in o) if (o[k] === undefined) delete o[k]
+    return o
+}
+
 export function assert(
     cond: boolean,
     msg = "Assertion failed",

packages/sample/src/basic.genai.mts

@@ -0,0 +1,16 @@
+script({
+    title: "Basic Prompt",
+    description: "A basic prompt that uses the chat API to answer questions",
+    parameters: {
+        question: {
+            type: "string",
+            default: "Who is the most famous person in the world?",
+        },
+    },
+})
+
+$`You are an AI assistant who helps people find information.
+As the assistant, you answer questions briefly, succinctly.`
+$`${env.vars.question}
+
+${env.vars.hint}`

packages/sample/src/cli.test.ts

@@ -115,6 +115,14 @@ describe("retrieval", () => {
     })
 })
 
+describe("prompty", () => {
+    const cmd = "prompty"
+    test("src", async () => {
+        const res = await $`node ${cli} ${cmd} "src/*.prompty"`.nothrow()
+        assert(!res.exitCode)
+    })
+})
+
 describe("workspace", () => {
     const cmd = "workspace"
     describe("grep", () => {