support for system prompts in runPrompt

.vscode/settings.json

@@ -25,8 +25,10 @@
         "millis",
         "ollama",
         "openai",
+        "promptdom",
         "promptfoo",
         "stringifying",
+        "sysr",
         "treesitter",
         "typecheck",
         "vsix",

packages/core/src/chat.ts

@@ -8,7 +8,7 @@
     CancellationToken,
     checkCancelled,
 } from "./cancellation"
-import { assert, logError } from "./util"
+import { assert, logError, logVerbose } from "./util"
 import { extractFenced, findFirstDataFence } from "./fence"
 import {
     toStrictJSONSchema,
@@ -558,6 +558,7 @@
                     if (resp.variables)
                         genVars = { ...(genVars || {}), ...resp.variables }
                 } finally {
+                    logVerbose("")
                     trace.endDetails()
                 }
 

packages/core/src/expander.ts

@@ -27,15 +27,16 @@
 } from "./chattypes"
 import { promptParametersSchemaToJSONSchema } from "./parameters"
 
-async function callExpander(
+export async function callExpander(
+    prj: Project,
     r: PromptScript,
     vars: ExpansionVariables,
     trace: MarkdownTrace,
     options: GenerationOptions
 ) {
     assert(!!options.model)
     const { provider, model } = parseModelIdentifier(r.model ?? options.model)
-    const ctx = await createPromptContext(vars, trace, options, model)
+    const ctx = await createPromptContext(prj, vars, trace, options, model)
 
     let status: GenerationStatus = undefined
     let statusText: string = undefined
@@ -190,7 +191,7 @@
         normalizeInt(env.vars["maxToolCalls"]) ??
         normalizeInt(env.vars["max_tool_calls"]) ??
         template.maxToolCalls ??
-        MAX_TOOL_CALLS        
+        MAX_TOOL_CALLS
     let seed = options.seed ?? normalizeInt(env.vars["seed"]) ?? template.seed
     if (seed !== undefined) seed = seed >> 0
 
@@ -206,7 +207,7 @@
     trace.startDetails("🧬 prompt")
     trace.detailsFenced("📓 script source", template.jsSource, "js")
 
-    const prompt = await callExpander(template, env, trace, options)
+    const prompt = await callExpander(prj, template, env, trace, options)
 
     const images = prompt.images
     const schemas = prompt.schemas
@@ -223,7 +224,11 @@
 
     if (prompt.status !== "success" || prompt.text === "")
         // cancelled
-        return { status: prompt.status, statusText: prompt.statusText, messages }
+        return {
+            status: prompt.status,
+            statusText: prompt.statusText,
+            messages,
+        }
 
     if (cancellationToken?.isCancellationRequested)
         return { status: "cancelled", statusText: "user cancelled", messages }
@@ -238,21 +243,17 @@
 
     for (let i = 0; i < systems.length; ++i) {
         if (cancellationToken?.isCancellationRequested)
-            return { status: "cancelled", statusText: "user cancelled", messages }
-
-        let systemTemplate = systems[i]
-        let system = prj.getTemplate(systemTemplate)
-        if (!system) {
-            if (systemTemplate) trace.error(`\`${systemTemplate}\` not found\n`)
-            if (i > 0) continue
-            systemTemplate = "system"
-            system = prj.getTemplate(systemTemplate)
-            assert(!!system)
-        }
+            return {
+                status: "cancelled",
+                statusText: "user cancelled",
+                messages,
+            }
 
-        trace.startDetails(`👾 ${systemTemplate}`)
+        const system = prj.getTemplate(systems[i])
+        if (!system) throw new Error(`system template ${systems[i]} not found`)
 
-        const sysr = await callExpander(system, env, trace, options)
+        trace.startDetails(`👾 ${system.id}`)
+        const sysr = await callExpander(prj, system, env, trace, options)
 
         if (sysr.images) images.push(...sysr.images)
         if (sysr.schemas) Object.assign(schemas, sysr.schemas)
@@ -272,12 +273,15 @@
             trace.fence(sysr.aici, "yaml")
             messages.push(sysr.aici)
         }
-
         trace.detailsFenced("js", system.jsSource, "js")
         trace.endDetails()
 
         if (sysr.status !== "success")
-            return { status: sysr.status, statusText: sysr.statusText, messages }
+            return {
+                status: sysr.status,
+                statusText: sysr.statusText,
+                messages,
+            }
     }
 
     const responseSchema = promptParametersSchemaToJSONSchema(

packages/core/src/promptcontext.ts

@@ -17,6 +17,7 @@
 import { createParsers } from "./parsers"
 import { readText } from "./fs"
 import {
+    PromptImage,
     PromptNode,
     appendChild,
     createFileMergeNode,
@@ -32,30 +33,42 @@
 } from "./runpromptcontext"
 import { CSVParse, CSVToMarkdown } from "./csv"
 import { INIParse, INIStringify } from "./ini"
-import { CancelError, isCancelError, serializeError } from "./error"
+import {
+    CancelError,
+    isCancelError,
+    NotSupportedError,
+    serializeError,
+} from "./error"
 import { createFetch } from "./fetch"
 import { XMLParse } from "./xml"
 import { GenerationOptions } from "./generation"
 import { fuzzSearch } from "./fuzzsearch"
 import { parseModelIdentifier } from "./models"
 import { renderAICI } from "./aici"
-import { MODEL_PROVIDER_AICI } from "./constants"
+import { MODEL_PROVIDER_AICI, SYSTEM_FENCE } from "./constants"
 import { JSONLStringify, JSONLTryParse } from "./jsonl"
 import { grepSearch } from "./grep"
 import { resolveFileContents, toWorkspaceFile } from "./file"
 import { vectorSearch } from "./vectorsearch"
-import { ChatCompletionMessageParam } from "./chattypes"
+import {
+    ChatCompletionMessageParam,
+    ChatCompletionSystemMessageParam,
+} from "./chattypes"
 import { resolveModelConnectionInfo } from "./models"
 import { resolveLanguageModel } from "./lm"
+import { callExpander } from "./expander"
+import { Project } from "./ast"
 
 export async function createPromptContext(
+    prj: Project,
     vars: ExpansionVariables,
     trace: MarkdownTrace,
     options: GenerationOptions,
     model: string
 ) {
     const { cancellationToken, infoCb } = options || {}
-    const env = structuredClone(vars)
+    const { generator, ...varsNoGenerator } = vars
+    const env = { generator, ...structuredClone(varsNoGenerator) }
     const parsers = await createParsers({ trace, model })
     const YAML = Object.freeze<YAML>({
         stringify: YAMLStringify,
@@ -246,7 +259,7 @@
         },
         runPrompt: async (generator, runOptions): Promise<RunPromptResult> => {
             try {
-                const { label } = runOptions || {}
+                const { label, system = [] } = runOptions || {}
                 trace.startDetails(`🎁 run prompt ${label || ""}`)
                 infoCb?.({ text: `run prompt ${label || ""}` })
 
@@ -263,6 +276,7 @@
                 let tools: ToolCallback[] = undefined
                 let schemas: Record<string, JSONSchema> = undefined
                 let chatParticipants: ChatParticipant[] = undefined
+
                 // expand template
                 const { provider } = parseModelIdentifier(genOptions.model)
                 if (provider === MODEL_PROVIDER_AICI) {
@@ -289,17 +303,69 @@
                         throw new Error("errors while running prompt")
                 }
 
+                const systemMessage: ChatCompletionSystemMessageParam = {
+                    role: "system",
+                    content: "",
+                }
+                for (const systemId of system) {
+                    checkCancelled(cancellationToken)
+
+                    const system = prj.getTemplate(systemId)
+                    if (!system)
+                        throw new Error(`system template ${systemId} not found`)
+                    trace.startDetails(`👾 ${system.id}`)
+                    const sysr = await callExpander(
+                        prj,
+                        system,
+                        env,
+                        trace,
+                        options
+                    )
+                    if (sysr.images?.length)
+                        throw new NotSupportedError("images")
+                    if (sysr.schemas) Object.assign(schemas, sysr.schemas)
+                    if (sysr.functions) tools.push(...sysr.functions)
+                    if (sysr.fileMerges?.length)
+                        throw new NotSupportedError("fileMerges")
+                    if (sysr.outputProcessors?.length)
+                        throw new NotSupportedError("outputProcessors")
+                    if (sysr.chatParticipants)
+                        chatParticipants.push(...sysr.chatParticipants)
+                    if (sysr.fileOutputs?.length)
+                        throw new NotSupportedError("fileOutputs")
+                    if (sysr.logs?.length)
+                        trace.details("📝 console.log", sysr.logs)
+                    if (sysr.text) {
+                        systemMessage.content +=
+                            SYSTEM_FENCE + "\n" + sysr.text + "\n"
+                        trace.fence(sysr.text, "markdown")
+                    }
+                    if (sysr.aici) {
+                        trace.fence(sysr.aici, "yaml")
+                        messages.push(sysr.aici)
+                    }
+                    trace.detailsFenced("js", system.jsSource, "js")
+                    trace.endDetails()
+                    if (sysr.status !== "success")
+                        throw new Error(
+                            `system ${system.id} failed ${sysr.status} ${sysr.statusText}`
+                        )
+                }
+                if (systemMessage.content) messages.unshift(systemMessage)
+
                 const connection = await resolveModelConnectionInfo(
                     genOptions,
                     { trace, token: true }
                 )
+                checkCancelled(cancellationToken)
                 if (!connection.configuration)
                     throw new Error(
                         "model connection error " + connection.info?.model
                     )
                 const { completer } = await resolveLanguageModel(
                     connection.configuration.provider
                 )
+                checkCancelled(cancellationToken)
                 if (!completer)
                     throw new Error(
                         "model driver not found for " + connection.info

packages/core/src/runpromptcontext.ts

@@ -15,7 +15,6 @@ import {
 import { MarkdownTrace } from "./trace"
 import { GenerationOptions } from "./generation"
 import { promptParametersSchemaToJSONSchema } from "./parameters"
-import { isJSONSchema } from "./schema"
 import { consoleLogFormat } from "./logging"
 import { resolveFileDataUri } from "./file"
 import { isGlobMatch } from "./glob"

packages/core/src/systems.ts

@@ -6,22 +6,24 @@ export function resolveSystems(prj: Project, template: PromptScript) {
     const systems = Array.from(template.system ?? []).slice(0)
 
     if (template.system === undefined) {
-        const useSchema = /defschema/i.test(jsSource)
+        const useSchema = /\Wdefschema\W/i.test(jsSource)
         if (!template.responseType) {
             systems.push("system")
             systems.push("system.explanations")
         }
         // select file expansion type
-        if (/diff/i.test(jsSource)) systems.push("system.diff")
-        else if (/changelog/i.test(jsSource)) systems.push("system.changelog")
-        else if (/file/i.test(jsSource)) {
+        if (/\Wdiff\W/i.test(jsSource)) systems.push("system.diff")
+        else if (/\Wchangelog\W/i.test(jsSource))
+            systems.push("system.changelog")
+        else if (/\Wfile\W/i.test(jsSource)) {
             systems.push("system.files")
             if (useSchema) systems.push("system.files_schema")
         }
         if (useSchema) systems.push("system.schema")
-        if (/annotation|warning|error/i.test(jsSource))
+        if (/\W(annotation|warning|error)\W/i.test(jsSource))
             systems.push("system.annotations")
-        if (/diagram|chart/i.test(jsSource)) systems.push("system.diagrams")
+        if (/\W(diagram|chart)\W/i.test(jsSource))
+            systems.push("system.diagrams")
     }
 
     if (template.tools?.length)

packages/core/src/types/prompt_template.d.ts

@@ -1283,6 +1283,11 @@ interface PromptGeneratorOptions extends ModelOptions {
      * Label for trace
      */
     label?: string
+
+    /**
+     * List of system prompts if any
+     */
+    system?: SystemPromptId[]
 }
 
 interface FileOutputOptions {