refcatoring assembly of messages

.github/workflows/genai-pr-commit-review.yml

@@ -30,6 +30,7 @@ jobs:
               run: git fetch origin && git pull origin main:main
             - name: genaiscript pr-review-commit
               run: node packages/cli/built/genaiscript.cjs run pr-review-commit --out ./temp/genai/pr-review-commit -prr --out-trace $GITHUB_STEP_SUMMARY
+              continue-on-error: true
               env:
                   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
                   GITHUB_COMMIT_SHA: ${{ github.event.pull_request.head.sha}}

.github/workflows/genai-pr-docs-commit-review.yml

@@ -25,6 +25,7 @@ jobs:
               run: git fetch origin && git pull origin main:main
             - name: genaiscript pr-review-commit
               run: node packages/cli/built/genaiscript.cjs run pr-docs-review-commit --out ./temp/genai/pr-docs-review-commit -prr --out-trace $GITHUB_STEP_SUMMARY
+              continue-on-error: true
               env:
                   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
                   GITHUB_COMMIT_SHA: ${{ github.event.pull_request.head.sha}}

.github/workflows/ollama.yml

@@ -31,6 +31,6 @@ jobs:
             - name: download ollama docker
               run: docker run -d -v ollama:/root/.ollama -p 11434:11434 --name ollama ollama/ollama
             - name: run summarize-ollama-phi3
-              run: yarn test:summarize --model ollama:phi3 --out ./temp/summarize-ollama-phi3
+              run: yarn test:summarize --model ollama:phi3.5 --out ./temp/summarize-ollama-phi3
 #            - name: run vector-search
 #              run: yarn run:script vector-search --model ollama:phi3 --out ./temp/rag

.github/workflows/openai.yml

@@ -1,22 +1,22 @@
 name: openai smoke tests
 on:
     workflow_dispatch:
-    pull_request:
-        paths:
-            - yarn.lock
-            - ".github/workflows/github-models.yml"
-            - "packages/core/**/*"
-            - "packages/cli/**/*"
-            - "packages/samples/**/*"
-    push:
-        branches:
-            - main
-        paths:
-            - yarn.lock
-            - ".github/workflows/github-models.yml"
-            - "packages/core/**/*"
-            - "packages/cli/**/*"
-            - "packages/samples/**/*"
+    # pull_request:
+    #     paths:
+    #         - yarn.lock
+    #         - ".github/workflows/github-models.yml"
+    #         - "packages/core/**/*"
+    #         - "packages/cli/**/*"
+    #         - "packages/samples/**/*"
+    # push:
+    #     branches:
+    #         - main
+    #     paths:
+    #         - yarn.lock
+    #         - ".github/workflows/github-models.yml"
+    #         - "packages/core/**/*"
+    #         - "packages/cli/**/*"
+    #         - "packages/samples/**/*"
 concurrency:
     group: ${{ github.workflow }}-${{ github.ref }}-github-models
     cancel-in-progress: true

.github/workflows/playwright.yml

@@ -35,9 +35,7 @@ jobs:
               run: yarn typecheck
             - name: compile
               run: yarn compile
+            - name: download ollama docker
+              run: docker run -d -v ollama:/root/.ollama -p 11434:11434 --name ollama ollama/ollama
             - name: run browse-text
-              run: yarn run:script browse-text --out ./temp/browse-text
-              env:
-                  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-                  OPENAI_API_TYPE: ${{ secrets.OPENAI_API_TYPE }}
-                  OPENAI_API_BASE: ${{ secrets.OPENAI_API_BASE }}
+              run: yarn run:script browse-text --out ./temp/browse-text --model ollama:phi3.5

docs/src/content/docs/reference/scripts/system.mdx

@@ -2978,7 +2978,15 @@ JSON schemas can also be applied to YAML or TOML files.
     ...
     \`\`\`
 
+
+## Code section with Schema
+
+When you generate JSON or YAML or CSV code section according to a named schema, 
+you MUST add the schema identifier in the code fence header.
 `
+
+fence("...", { language: "json", schema: "<schema-identifier>" })
+
 `````
 
 

docs/yarn.lock

@@ -5003,9 +5003,9 @@ yocto-queue@^1.1.1:
   integrity sha512-b4JR1PFR10y1mKjhHY9LaGo6tmrgjit7hxVIeAmyMw3jegXR4dhYqLaQF5zMXZxY7tLpMyJeLjr1C4rLmkVe8g==
 
 zod-to-json-schema@^3.23.3:
-  version "3.23.4"
-  resolved "https://registry.yarnpkg.com/zod-to-json-schema/-/zod-to-json-schema-3.23.4.tgz#f6fa99a15412ac2c79721ad1b9d901ecfb11575b"
-  integrity sha512-LjSQ9WT8qkvb4OHlkC3AlizvC+pwKIxpBohK4FLzCm7RyJES8uLpHP6BWljCtBjBXJ9Xa1R9IGZcICuSf8xl2Q==
+  version "3.23.5"
+  resolved "https://registry.yarnpkg.com/zod-to-json-schema/-/zod-to-json-schema-3.23.5.tgz#ec23def47dcafe3a4d640eba6a346b34f9a693a5"
+  integrity sha512-5wlSS0bXfF/BrL4jPAbz9da5hDlDptdEppYfe+x4eIJ7jioqKG9uUxOwPzqof09u/XeVdrgFu29lZi+8XNDJtA==
 
 zod-to-ts@^1.2.0:
   version "1.2.0"

package.json

@@ -83,7 +83,7 @@
     "devDependencies": {
         "@inquirer/prompts": "^7.0.0",
         "glob": "^11.0.0",
-        "npm-check-updates": "^17.1.4",
+        "npm-check-updates": "^17.1.5",
         "prettier": "^3.3.3",
         "zx": "^8.1.9"
     }

packages/cli/package.json

@@ -76,15 +76,15 @@
     "commander": "^12.1.0",
     "diff": "^7.0.0",
     "dotenv": "^16.4.5",
-    "es-toolkit": "^1.25.2",
+    "es-toolkit": "^1.26.0",
     "esbuild": "^0.24.0",
     "execa": "^9.4.1",
     "fs-extra": "^11.2.0",
     "glob": "^11.0.0",
     "memorystream": "^0.3.1",
     "node-sarif-builder": "^3.2.0",
     "octokit": "^4.0.2",
-    "openai": "^4.68.3",
+    "openai": "^4.68.4",
     "pretty-bytes": "^6.1.1",
     "replace-ext": "^2.0.0",
     "ws": "^8.18.0",

packages/cli/src/playwright.ts

@@ -40,7 +40,7 @@ export class BrowserManager {
             undefined,
             "npx",
             [
-                "--yes,",
+                "--yes",
                 `playwright@${PLAYWRIGHT_VERSION}`,
                 "install",
                 "--with-deps",

packages/core/package.json

@@ -41,7 +41,7 @@
     "csv-stringify": "^6.5.1",
     "diff": "^7.0.0",
     "dotenv": "^16.4.5",
-    "es-toolkit": "^1.25.2",
+    "es-toolkit": "^1.26.0",
     "esbuild": "^0.24.0",
     "fast-xml-parser": "^4.5.0",
     "fetch-retry": "^6.0.0",
@@ -63,7 +63,7 @@
     "minimatch": "^10.0.1",
     "minisearch": "^7.1.0",
     "mustache": "^4.2.0",
-    "openai": "^4.68.3",
+    "openai": "^4.68.4",
     "p-limit": "^6.1.0",
     "parse-diff": "^0.11.1",
     "prettier": "^3.3.3",

packages/core/src/chat.ts

@@ -24,6 +24,7 @@ import {
     MAX_DATA_REPAIRS,
     MAX_TOOL_CALLS,
     MAX_TOOL_CONTENT_TOKENS,
+    SYSTEM_FENCE,
 } from "./constants"
 import { parseAnnotations } from "./annotations"
 import { errorMessage, isCancelError, serializeError } from "./error"
@@ -32,10 +33,12 @@ import { createChatTurnGenerationContext } from "./runpromptcontext"
 import { dedent } from "./indent"
 import { traceLanguageModelConnection } from "./models"
 import {
+    ChatCompletionAssistantMessageParam,
     ChatCompletionContentPartImage,
     ChatCompletionMessageParam,
     ChatCompletionResponse,
     ChatCompletionsOptions,
+    ChatCompletionSystemMessageParam,
     ChatCompletionTool,
     ChatCompletionToolCall,
     ChatCompletionUserMessageParam,
@@ -564,17 +567,26 @@ async function processChatMessage(
                 const node = ctx.node
                 checkCancelled(cancellationToken)
                 // expand template
-                const { errors, userPrompt } = await renderPromptNode(
-                    options.model,
-                    node,
-                    {
+                const { errors, messages: participantMessages } =
+                    await renderPromptNode(options.model, node, {
                         flexTokens: options.flexTokens,
                         trace,
-                    }
-                )
-                if (userPrompt?.trim().length) {
-                    trace.detailsFenced(`💬 message`, userPrompt, "markdown")
-                    messages.push({ role: "user", content: userPrompt })
+                    })
+                if (participantMessages?.length) {
+                    if (
+                        participantMessages.some(
+                            ({ role }) => role === "system"
+                        )
+                    )
+                        throw new Error(
+                            "system messages not supported for chat participants"
+                        )
+                    renderMessagesToMarkdown(participantMessages)
+                    trace.details(
+                        `💬 messages (${participantMessages.length})`,
+                        renderMessagesToMarkdown(participantMessages)
+                    )
+                    messages.push(...participantMessages)
                     needsNewTurn = true
                 } else trace.item("no message")
                 if (errors?.length) {
@@ -785,3 +797,48 @@ export function tracePromptResult(trace: MarkdownTrace, resp: RunPromptResult) {
             "\n\n" + HTMLEscape(prettifyMarkdown(text)) + "\n\n"
         )
 }
+
+export function appendUserMessage(
+    messages: ChatCompletionMessageParam[],
+    content: string
+) {
+    if (!content) return
+    const last = messages.at(-1) as ChatCompletionUserMessageParam
+    if (last?.role === "user") last.content += content + "\n"
+    else
+        messages.push({
+            role: "user",
+            content,
+        } as ChatCompletionUserMessageParam)
+}
+
+export function appendAssistantMessage(
+    messages: ChatCompletionMessageParam[],
+    content: string
+) {
+    if (!content) return
+    const last = messages.at(-1) as ChatCompletionAssistantMessageParam
+    if (last?.role === "assistant") last.content += content
+    else
+        messages.push({
+            role: "assistant",
+            content,
+        } satisfies ChatCompletionAssistantMessageParam)
+}
+
+export function appendSystemMessage(
+    messages: ChatCompletionMessageParam[],
+    content: string
+) {
+    if (!content) return
+    let last = messages[0] as ChatCompletionSystemMessageParam
+    if (!last) {
+        last = {
+            role: "system",
+            content,
+        } as ChatCompletionSystemMessageParam
+        messages.unshift(last)
+    }
+    if (last.content) last.content += SYSTEM_FENCE
+    last.content += content
+}