Blog ideation

.vscode/settings.json

@@ -34,6 +34,7 @@
         "sysr",
         "treesitter",
         "typecheck",
+        "unfence",
         "vsix",
         "xpai"
     ],

docs/src/content/docs/reference/cli/commands.md

@@ -119,8 +119,8 @@
 Commands:
   list                      List all available scripts in workspace
   create <name>             Create a new script
   fix                       fix all definition files
-  compile                   Compile all script in workspace
+  compile [folders...]      Compile all script in workspace
   model [options] [script]  List model connection information for scripts
   help [command]            display help for command
 ```
@@ -164,10 +164,13 @@
 ### `scripts compile`
 
 ```
-Usage: genaiscript scripts compile [options]
+Usage: genaiscript scripts compile [options] [folders...]
 
 Compile all script in workspace
 
+Arguments:
+  folders     Pattern to match files
+
 Options:
   -h, --help  display help for command
 ```

genaisrc/blog-generator.genai.mjs

@@ -1,6 +1,7 @@
 script({
     description: "Generate a blog post for Dev.to from the documentation",
     model: "openai:gpt-4o",
+    system: [],
     tools: ["fs"],
     parameters: {
         theme: {
@@ -38,7 +39,7 @@ Use these files to help you generate a topic for the blog post.
 `
         },
         {
-            model: "openai:gpt-4o",
+            label: "generate theme",
             temperature: 1,
             system: [
                 "system.tools",
@@ -51,32 +52,99 @@ Use these files to help you generate a topic for the blog post.
     topic = res.text
 }
 
+// generate a working code snippet from topic
+let snippet
+{
+    const { text, fences, error } = await runPrompt(
+        (_) => {
+            _.defTool(
+                "genaiscript_typecheck",
+                "Validates the syntax and type checks a GenAIScript source code",
+                {
+                    source: {
+                        type: "string",
+                        description: "The GenAIScript javascript source code",
+                    },
+                },
+                async ({ source }) => {
+                    // write file to a scratch folder
+                    console.log(`checking source code`)
+                    await workspace.writeText(
+                        "temp/blog-post/script.genai.mjs",
+                        source
+                    )
+                    return await host.exec("node", [
+                        "packages/cli/built/genaiscript.cjs",
+                        "scripts",
+                        "compile",
+                        "temp/blog-post",
+                    ])
+                }
+            )
+
+            _.def("TOPIC", topic)
+            _.$`You are a JavaScript developer expert on GenAIScript (https://microsoft.github.io/genaiscript).
+
+        # Task
+
+        - Generate a GenAISCript source code that implements the idea described in TOPIC.
+        - Validate syntax and checking with genaiscript_typecheck.
+        - Respond ONLY with the JavaScript source code. Do NOT fence code in markdown. Do not add text around code.
+
+        # Information
+
+        Use these files to help you generate a topic for the blog post.
+
+        - the code will be executed in node.js by the GenAIScript CLI
+        - the genaiscript type definition: genaiscript/genaiscript.d.ts. Assume that all globals are ambient. Do not import or require genaiscript module.
+        - the genaiscript samples: packages/sample/src/*.genai.*
+        - the documentation: docs/src/content/docs/**/*.md*
+        - the online documentation: https://microsoft.github.io/genaiscript/
+        `
+        },
+        {
+            label: "generate script",
+            system: [
+                "system.tools",
+                "system.fs_find_files",
+                "system.fs_read_file",
+            ],
+            temperature: 0.5,
+        }
+    )
+    if (error) throw error
+    snippet =
+        fences.find(
+            ({ language }) => language === "js" || language === "javascript"
+        )?.content ?? text
+}
+
 // generate a blog post
 const today = new Date()
 const yyyy = today.getFullYear()
 const mm = String(today.getMonth() + 1).padStart(2, "0")
 const dd = String(today.getDate()).padStart(2, "0")
 const formattedDate = `${yyyy}-${mm}-${dd}`
 def("TOPIC", topic)
+def("SNIPPET", snippet)
 
 $`
 # Task
 
-Generate a blog post for the web site https://dev.to on the topic of using GenAIScript script 
-to solve the task described in TOPIC.
+Generate a blog post that explains how to write the code in SNIPPET.
 
 The main purpose is to create a genaiscript prompt generation script.
+Respond with the markdown content of the blog post.
 
 # Writing style and instructions
 
 - generate the blog post content, nothing else
-- save the generated markdown to a new file under the docs/src/content/docs/blog folder. THIS IS IMPORTANT
 - use a clear and engaging tone
-- illustrate with code examples
+- explain each line of code separately, link to the documentation if possible
 - title should be click-bait, use quotes (") around title
 - do NOT generate cover_image
 - the script will be execute by GenAIScript CLI
-- the TypeScript API is defined in the file genaisrc/genaiscript.d.ts
+- the TypeScript API is defined in the file genaisrc/genaiscript.d.ts. Assume that all globals are ambient. Do not import or require genaiscript module.
 - generate a single file for the blog post, do NOT generate other files
 - examples of GenAIScript code are in folder packages/sample/src/**/*.genai.*js
 - do NOT explain how to install GenAIScript or the GenAIScript CLI
@@ -97,17 +165,20 @@ You can extract information from the following files:
 - the online documentation: https://microsoft.github.io/genaiscript/
 
 `
-
-defFileOutput("docs/src/content/docs/blog/*.md", "The generated blog post")
 defOutputProcessor((output) => {
-    if (!Object.keys(output.fileEdits || {}).length) {
-        const fence = output.fences.find((f) => f.language === "markdown")
-        if (fence) {
-            const files = {
-                [`docs/src/content/docs/blog/unnamed-${formattedDate}.md`]:
-                    fence.content,
-            }
-            return { files }
-        }
+    let md = output.text
+    if (/\`\`\`markdown\n/.test(md)) {
+        md = md.replace(/\`\`\`markdown\n/g, "").replace(/\`\`\`\n?$/g, "")
+    }
+    const fm = parsers.frontmatter(md)
+    const fn =
+        `docs/src/content/docs/blog/${fm.title.replace(/[^a-z0-9]+/gi, "-")}.md`.toLocaleLowerCase()
+    const sn =
+        `packages/sample/genaisrc/blog/${fm.title.replace(/[^a-z0-9]+/gi, "-")}.genai.mjs`.toLocaleLowerCase()
+    return {
+        files: {
+            [fn]: md,
+            [sn]: snippet,
+        },
     }
 })

packages/cli/src/cli.ts

@@ -145,10 +145,13 @@
         .option("-cs, --csv-separator <string>", "csv separator", "\t")
         .option("-ae, --apply-edits", "apply file edits")
         .option(
             "--vars <namevalue...>",
             "variables, as name=value, stored in env.vars"
         )
-        .option("-rr, --run-retry <number>", "number of retries for the entire run")
+        .option(
+            "-rr, --run-retry <number>",
+            "number of retries for the entire run"
+        )
         .action(runScriptWithExitCode)
 
     const test = program.command("test")
@@ -205,6 +208,7 @@
     scripts
         .command("compile")
         .description("Compile all script in workspace")
+        .argument("[folders...]", "Pattern to match files")
         .action(compileScript)
     scripts
         .command("model")

packages/cli/src/scripts.ts

@@ -24,18 +24,25 @@ export async function createScript(name: string) {
     const t = coreCreateScript(name)
     const pr = await copyPrompt(t, { fork: false, name })
     console.log(`created script at ${pr}`)
-    await compileScript()
+    await compileScript([])
 }
 
 export async function fixScripts() {
     const project = await buildProject()
     await fixPromptDefinitions(project)
 }
 
-export async function compileScript() {
+export async function compileScript(folders: string[]) {
     const project = await buildProject()
     await fixPromptDefinitions(project)
-    for (const folder of project.folders()) {
+    const scriptFolders = project.folders()
+    const foldersToCompile = (
+        folders?.length ? folders : project.folders().map((f) => f.dirname)
+    )
+        .map((f) => scriptFolders.find((sf) => sf.dirname === f))
+        .filter((f) => f)
+
+    for (const folder of foldersToCompile) {
         const { dirname, js, ts } = folder
         logVerbose(`compiling ${dirname}`)
         if (js) {

packages/core/src/chat.ts

@@ -9,7 +9,7 @@ import {
     checkCancelled,
 } from "./cancellation"
 import { assert, logError, logVerbose } from "./util"
-import { extractFenced, findFirstDataFence } from "./fence"
+import { extractFenced, findFirstDataFence, unfence } from "./fence"
 import {
     toStrictJSONSchema,
     validateFencesWithSchema,
@@ -330,13 +330,21 @@ Repair the DATA_FORMAT_ISSUES. THIS IS IMPORTANT.`
     return true
 }
 
-function assistantText(messages: ChatCompletionMessageParam[]) {
+function assistantText(
+    messages: ChatCompletionMessageParam[],
+    responseType?: PromptTemplateResponseType
+) {
     let text = ""
     for (let i = messages.length - 1; i >= 0; i--) {
         const msg = messages[i]
         if (msg.role !== "assistant") break
         text = msg.content + text
     }
+
+    if (responseType === undefined) {
+        text = unfence(text, "(markdown|md)")
+    }
+
     return text
 }
 
@@ -352,7 +360,7 @@ function structurifyChatSession(
 ): RunPromptResult {
     const { trace, responseType, responseSchema } = options
     const { resp, err } = others || {}
-    const text = assistantText(messages)
+    const text = assistantText(messages, responseType)
     const annotations = parseAnnotations(text)
     const finishReason = isCancelError(err)
         ? "cancel"

packages/core/src/constants.ts

@@ -4,7 +4,7 @@ export const RECONNECT = "reconnect"
 export const OPEN = "open"
 export const MAX_CACHED_TEMPERATURE = 0.5
 export const MAX_CACHED_TOP_P = 0.5
-export const MAX_TOOL_CALLS = 100
+export const MAX_TOOL_CALLS = 10000
 
 // https://learn.microsoft.com/en-us/azure/ai-services/openai/reference
 // https://github.com/Azure/azure-rest-api-specs/blob/main/specification/cognitiveservices/data-plane/AzureOpenAI/inference/stable/2024-02-01/inference.yaml

packages/core/src/fence.ts

@@ -212,3 +212,12 @@ ${validation.error.split("\n").join("\n> ")}`
         )
         .join("\n")
 }
+
+export function unfence(text: string, language: string) {
+    const startRx = new RegExp(`^\`\`\`${language}\n`)
+    const endRx = /\n```[\n\s]*$/
+    if (startRx.test(text) && endRx.test(text)) {
+        return text.replace(startRx, "").replace(endRx, "")
+    }
+    return text
+}

packages/core/src/promptrunner.ts

@@ -323,6 +323,7 @@ export async function runTemplate(
                             trace.detailsFenced(`📁 file ${fn}`, content)
                             const fileEdit = await getFileEdit(fn)
                             fileEdit.after = content
+                            fileEdit.validation = { valid: true }
                         }
                     if (oannotations) annotations = oannotations.slice(0)
                 }

packages/core/src/types/prompt_template.d.ts

@@ -291,6 +291,10 @@ type PromptAssertion = {
 )
 
 interface PromptTest {
+    /**
+     * Short name of the test
+     */
+    name?: string
     /**
      * Description of the test.
      */
@@ -629,6 +633,7 @@ type PromptSystemArgs = Omit<
     | "maxTokens"
     | "seed"
     | "tests"
+    | "responseLanguage"
     | "responseType"
     | "responseSchema"
     | "files"