fix output processor

docs/src/content/docs/blog/real-time-data-analysis.md

@@ -0,0 +1,172 @@
+---
+title: "Unlocking Real-Time Data Analytics with GenAIScript Effortlessly"
+date: 2024-08-21
+authors: genaiscript
+tags: [GenAIScript, Data Analytics, Real-Time, Data Visualization, Automation, JavaScript, AI]
+draft: true
+---
+
+## Unlocking Real-Time Data Analytics with GenAIScript Effortlessly
+
+### Introduction
+As data becomes increasingly central to decision-making processes across industries, the ability to perform real-time data analytics and visualization becomes a critical capability. In this blog post, we will dive into how GenAIScript can be utilized to seamlessly integrate data analytics and visualization into your workflows.
+
+### 1. Real-Time Data Analytics with GenAIScript
+
+#### Setting up Real-Time Data Fetch
+GenAIScript can fetch data from various real-time data sources like APIs, databases, and IoT devices. Here’s an example script to fetch data from a public API:
+
+```typescript
+const apiResponse = await fetch("https://api.example.com/data");
+const jsonData = await apiResponse.json();
+
+def("API_DATA", jsonData, {
+    language: "json",
+    maxTokens: 5000,
+});
+```
+
+#### Data Pre-processing and Cleaning
+Before analysis, data often needs to be cleaned and pre-processed. GenAIScript can help automate this step:
+
+```typescript
+const cleanedData = API_DATA.map(item => {
+    return {
+        ...item,
+        value: item.value.trim(),
+        date: new Date(item.date)
+    };
+});
+
+def("CLEANED_DATA", cleanedData, {
+    language: "json",
+    maxTokens: 5000,
+});
+```
+
+#### Running Real-Time Analyses
+Performing on-the-fly data analytics, including statistical summaries and trend analysis, is straightforward with GenAIScript:
+
+```typescript
+const stats = {
+    mean: (CLEANED_DATA.reduce((acc, item) => acc + item.value, 0) / CLEANED_DATA.length),
+    min: Math.min(...CLEANED_DATA.map(item => item.value)),
+    max: Math.max(...CLEANED_DATA.map(item => item.value))
+};
+
+def("STATS", stats, {
+    language: "json",
+    maxTokens: 500,
+});
+```
+
+### 2. Integrating External Tools and APIs
+
+#### Connecting to Data APIs
+Fetch and analyze data from public or enterprise APIs:
+
+```typescript
+const weatherResponse = await fetch("https://api.weather.com/v3/wx/conditions/current?apiKey=YOUR_API_KEY&format=json");
+const weatherData = await weatherResponse.json();
+
+def("WEATHER_DATA", weatherData, {
+    language: "json",
+    maxTokens: 5000,
+});
+```
+
+#### Leveraging Third-Party Analytical Tools
+Integrate tools like Pandas, NumPy, or machine learning libraries for enhanced analytics:
+
+```typescript
+import { python } from "genaiscript";
+const pandasScript = `
+import pandas as pd
+data = pd.read_json("CLEANED_DATA.json")
+summary = data.describe()
+summary.to_json()
+`;
+
+const summaryStats = await python.run(pandasScript);
+def("SUMMARY_STATS", summaryStats, {
+    language: "json",
+    maxTokens: 5000,
+});
+```
+
+### 3. Data Visualization
+
+#### Creating Dynamic Visualizations
+Using visualization libraries like D3.js or Chart.js with GenAIScript to produce charts and graphs:
+
+```typescript
+import { Chart } from "chart.js";
+
+const ctx = document.getElementById("myChart");
+const myChart = new Chart(ctx, {
+    type: 'line',
+    data: {
+        labels: CLEANED_DATA.map(item => item.date),
+        datasets: [{
+            label: 'Value',
+            data: CLEANED_DATA.map(item => item.value),
+            borderColor: 'rgba(75, 192, 192, 1)',
+            borderWidth: 1
+        }]
+    }
+});
+```
+
+#### Interpreting and Displaying Results
+Convert analytical results into insightful visualizations that can be easily interpreted:
+
+```typescript
+const visualization = `
+<canvas id="myChart" width="400" height="200"></canvas>
+<script>
+    ${myChart}
+</script>
+`;
+
+document.body.innerHTML = visualization;
+```
+
+### 4. Automating Data-Driven Decision Making
+
+#### Automated Reporting
+Generate and distribute real-time analytical reports:
+
+```typescript
+const report = `
+<h1>Real-Time Data Analytics Report</h1>
+<p>Mean Value: ${STATS.mean}</p>
+<p>Min Value: ${STATS.min}</p>
+<p>Max Value: ${STATS.max}</p>
+<canvas id="reportChart" width="400" height="200"></canvas>
+<script>
+    ${myChart}
+</script>
+`;
+
+sendEmail("[email protected]", "Real-Time Data Analytics Report", report);
+```
+
+#### Alerting and Anomaly Detection
+Set up alerts for significant data changes or anomalies detected in real-time:
+
+```typescript
+const threshold = 100;
+const anomalies = CLEANED_DATA.filter(item => item.value > threshold);
+
+if (anomalies.length > 0) {
+    sendAlert("Anomalies detected", `There are ${anomalies.length} anomalies exceeding the threshold of ${threshold}.`);
+}
+```
+
+### Conclusion
+Integrating real-time data analytics and visualization with GenAIScript can transform decision-making processes to be faster and more data-driven. Explore the advanced analytical capabilities of GenAIScript and start integrating it into your workflows today.
+
+### Call to Action
+Try out the example scripts, integrate your own data sources, and share your experiences and insights with the GenAIScript community.
+
+Happy coding and analyzing! 📊🚀

genaisrc/blog-generator.genai.mjs

@@ -5,20 +5,21 @@ script({
     parameters: {
         theme: {
             type: "string",
-            description: "The theme of the blog post"
+            description: "The theme of the blog post",
         },
         topic: {
             type: "string",
-            description: "The topic and goal of the article"
-        }
-    }
+            description: "The topic and goal of the article",
+        },
+    },
 })
 let { topic, theme } = env.vars
 
 if (!topic) {
     // step 1 generate a topic
-    const res = await runPrompt(_ => {
-        _.$`You are a blog writer expert on GenAIScript (https://microsoft.github.io/genaiscript).
+    const res = await runPrompt(
+        (_) => {
+            _.$`You are a blog writer expert on GenAIScript (https://microsoft.github.io/genaiscript).
 # Task
 
 Generate a blog post topic on the topic of writing and using a GenAIScript script.
@@ -34,17 +35,27 @@ Use these files to help you generate a topic for the blog post.
 - the existing blog posts: docs/src/content/docs/blog/*.md*
 - the online documentation: https://microsoft.github.io/genaiscript/
 `
-    }, { model: "openai:gpt-4o", system: ["system.tools", "system.fs_find_files", "system.fs_read_file"] })
+        },
+        {
+            model: "openai:gpt-4o",
+            temperature: 1,
+            system: [
+                "system.tools",
+                "system.fs_find_files",
+                "system.fs_read_file",
+            ],
+        }
+    )
     if (res.error) throw res.error
     topic = res.text
 }
 
 // generate a blog post
-const today = new Date();
-const yyyy = today.getFullYear();
-const mm = String(today.getMonth() + 1).padStart(2, '0');
-const dd = String(today.getDate()).padStart(2, '0');
-const formattedDate = `${yyyy}-${mm}-${dd}`;
+const today = new Date()
+const yyyy = today.getFullYear()
+const mm = String(today.getMonth() + 1).padStart(2, "0")
+const dd = String(today.getDate()).padStart(2, "0")
+const formattedDate = `${yyyy}-${mm}-${dd}`
 def("TOPIC", topic)
 
 $`
@@ -87,3 +98,14 @@ You can extract information from the following files:
 `
 
 defFileOutput("docs/src/content/docs/blog/*.md", "The generated blog post")
+defOutputProcessor(output => {
+    if (!Object.keys(output.fileEdits || {}).length) {
+        const fence = output.fences.find(f => f.language === "markdown")
+        if (fence) {
+            const files = {
+                [`docs/src/content/docs/blog/unnamed-${formattedDate}.md`]: fence.content
+            }
+            return { files }
+        }
+    }
+})

packages/cli/src/run.ts

@@ -1,7 +1,7 @@
 import { capitalize } from "inflection"
 import { resolve, join, relative, dirname } from "node:path"
 import { isQuiet } from "./log"
-import { emptyDir, ensureDir } from "fs-extra"
+import { emptyDir, ensureDir, appendFileSync } from "fs-extra"
 import { convertDiagnosticsToSARIF } from "./sarif"
 import { buildProject } from "./build"
 import { diagnosticsToCSV } from "../../core/src/ast"
@@ -62,17 +62,17 @@ import {
     azureDevOpsUpdatePullRequestDescription,
 } from "../../core/src/azuredevops"
 import { resolveTokenEncoder } from "../../core/src/encoders"
-import { appendFile, writeFile } from "fs/promises"
+import { writeFile } from "fs/promises"
 
 async function setupTraceWriting(trace: MarkdownTrace, filename: string) {
     logVerbose(`writing trace to ${filename}`)
     await ensureDir(dirname(filename))
     await writeFile(filename, "", { encoding: "utf-8" })
     trace.addEventListener(
         TRACE_CHUNK,
-        async (ev) => {
+        (ev) => {
             const tev = ev as TraceChunkEvent
-            await appendFile(filename, tev.chunk, { encoding: "utf-8" })
+            appendFileSync(filename, tev.chunk, { encoding: "utf-8" })
         },
         false
     )
@@ -163,7 +163,7 @@ export async function runScript(
         if (removeOut) await emptyDir(out)
         await ensureDir(out)
     }
-    if (outTrace && /^false$/i.test(outTrace) && trace)
+    if (outTrace && !/^false$/i.test(outTrace) && trace)
         await setupTraceWriting(trace, outTrace)
     if (out && trace) {
         const ofn = join(out, "res.trace.md")

packages/core/src/promptrunner.ts

@@ -1,7 +1,7 @@
 import { executeChatSession, tracePromptResult } from "./chat"
 import { Project, PromptScript } from "./ast"
 import { stringToPos } from "./parser"
-import { arrayify, assert, logVerbose, relativePath } from "./util"
+import { arrayify, assert, logError, logVerbose, relativePath } from "./util"
 import { runtimeHost } from "./host"
 import { applyLLMDiff, applyLLMPatch, parseLLMDiffs } from "./diff"
 import { MarkdownTrace } from "./trace"
@@ -327,6 +327,7 @@ export async function runTemplate(
                     if (oannotations) annotations = oannotations.slice(0)
                 }
             } catch (e) {
+                logError(e)
                 trace.error(`output processor failed`, e)
             } finally {
                 trace.endDetails()