Update defImages function to support multiple file types and handle B…

…lob conversion (#705) * Update defImages function to support multiple file types and handle Blob conversion * add azure blob sample * Renamed file and updated default parameters and logging in azure-blobs.genai.mts * Update Azure Blob Storage guide and script for image processing and summarization * Update frontmatter support for mdx, enhance blog drafts, and improve documentation details across various guides. * Update function name to defImages in images.md documentation
microsoft · Sep 11, 2024 · 9786584 · 9786584
1 parent 2148bde
commit 9786584
Show file tree

Hide file tree

Showing 37 changed files with 365 additions and 46 deletions.
diff --git a/docs/genaisrc/genaiscript.d.ts b/docs/genaisrc/genaiscript.d.ts
diff --git a/docs/package.json b/docs/package.json
@@ -14,7 +14,7 @@
     "preview": "astro preview",
     "astro": "astro",
     "genai:test": "node ../packages/cli/built/genaiscript.cjs test src/**/*.md",
-    "genai:frontmatter": "node ../packages/cli/built/genaiscript.cjs run frontmatter \"src/**/*.md\" --apply-edits",
+    "genai:frontmatter": "node ../packages/cli/built/genaiscript.cjs run frontmatter \"src/**/*.{md,mdx}\" --apply-edits",
     "genai:technical": "for file in \"src/**/*.md\"; do\nnode ../packages/cli/built/genaiscript.cjs run technical \"$file\" --apply-edits\ndone",
     "genai:alt-text": "node scripts/image-alt-text.mjs"
   },

diff --git a/...triage-with-genaiscript-harnessing-ai-to-prioritize-and-manage-github-issues.md b/...triage-with-genaiscript-harnessing-ai-to-prioritize-and-manage-github-issues.md
@@ -10,6 +10,9 @@ tags:
   - Project Management
   - Automation
   - GenAIScript
+description: Employ GenAIScript to automate GitHub issue triage with AI,
+  improving project management and issue resolution efficiency.
+keywords: GitHub issues, AI, project management, automation, issue triage
 
 ---
 

diff --git a/...s/unlock-devops-superpowers-automate-testing-and-deployment-with-genaiscript.md b/...s/unlock-devops-superpowers-automate-testing-and-deployment-with-genaiscript.md
@@ -9,6 +9,9 @@ tags:
   - Automation
   - Testing
   - Deployment
+description: Streamline your DevOps pipeline with GenAIScript by automating
+  testing and deployment through a tutorial script.
+keywords: DevOps, automation, testing, deployment, scripting
 
 ---
 

diff --git a/...drafts/unlock-the-power-of-ai-enhance-your-debugging-skills-with-genaiscript.md b/...drafts/unlock-the-power-of-ai-enhance-your-debugging-skills-with-genaiscript.md
@@ -8,6 +8,9 @@ tags:
   - Programming
   - Efficiency
 draft: true
+description: Leverage AI with GenAIScript to improve your debugging skills,
+  featuring a practical example for an efficient workflow.
+keywords: AI, debugging, programming efficiency, developer tools, GenAIScript
 
 ---
 

diff --git a/...ontent/docs/blog/drafts/unlock-the-power-of-dynamic-content-with-genaiscript.md b/...ontent/docs/blog/drafts/unlock-the-power-of-dynamic-content-with-genaiscript.md
@@ -8,6 +8,9 @@ tags:
   - Dynamic Content Generation
   - AI
   - Automation
+description: Discover how GenAIScript can automate dynamic content generation
+  using AI, with an easy-to-understand script example.
+keywords: dynamic content, automation, AI, content generation, scripting
 
 ---
 

diff --git a/...k-the-power-of-test-automation-mastering-genaiscript-for-streamlined-testing.md b/...k-the-power-of-test-automation-mastering-genaiscript-for-streamlined-testing.md
@@ -9,6 +9,10 @@ tags:
   - Test Automation
   - Software Development
   - Best Practices
+description: Explore test automation using GenAIScript to enhance and streamline
+  your testing processes with structured script breakdown.
+keywords: test automation, scripting, software testing, streamlined process,
+  developer tools
 
 ---
 

diff --git a/docs/src/content/docs/case-studies/release-notes.mdx b/docs/src/content/docs/case-studies/release-notes.mdx
@@ -1,7 +1,10 @@
 ---
 title: Release Notes
 sidebar:
-    order: 7
+  order: 7
+description: Generate comprehensive release notes combining commit history and code diffs
+keywords: release notes, automation, commit history, code diffs, release management
+
 ---
 import { Code } from '@astrojs/starlight/components';
 import scriptSrc from "../../../../../packages/sample/genaisrc/git-release-notes.genai?raw"

diff --git a/docs/src/content/docs/guides/auto-git-commit-message.mdx b/docs/src/content/docs/guides/auto-git-commit-message.mdx
@@ -1,8 +1,11 @@
 ---
-title: "Automated Git Commit Messages"
-keywords: ["GenAI", "Git", "Automation"]
+title: Automated Git Commit Messages
+keywords: Git, commit automation, software development, CLI, husky
 sidebar:
-    order: 15
+  order: 15
+description: Streamline your Git workflow with an automation script for
+  generating commit messages
+
 ---
 
 In the world of software development, making consistent and informative commit messages is crucial but often overlooked.

diff --git a/docs/src/content/docs/guides/images-in-azure-blob-storage.mdx b/docs/src/content/docs/guides/images-in-azure-blob-storage.mdx
@@ -0,0 +1,100 @@
+---
+title: Images in Azure Blob Storage
+sidebar:
+  order: 55
+description: Leverage Azure SDK to handle image files in Blob Storage within prompts
+keywords: Azure Blob Storage, Azure SDK, image handling, Node.js Buffer, inline prompts
+
+---
+
+import { Code } from '@astrojs/starlight/components';
+import { Steps } from "@astrojs/starlight/components"
+import source from "../../../../../packages/sample/genaisrc/azure-blobs.genai.mts?raw"
+
+It is possible to use the Azure Node.JS SDK to download images from Azure Blog Storage
+and use them in the prompt. The `defImages` function support the node.js [Buffer] type.
+
+## Configuration
+
+Install the [@azure/storage-blob](https://www.npmjs.com/package/@azure/storage-blob) and [@azure/identity](https://www.npmjs.com/package/@azure/identity) packages.
+
+```sh
+npm install -D @azure/storage-blob @azure/identity
+```
+
+Make sure to login with the Azure CLI and set the subscription.
+
+```sh
+az login
+```
+
+## Reading blobs
+
+Open a connection to the Azure Blob Storage and get a client to the container.
+We deconstruct the `account` and `container` from the `env.vars` object
+so that they can be set through the [cli](/genaiscript/reference/cli).
+
+```ts
+import { BlobServiceClient } from "@azure/storage-blob"
+import { DefaultAzureCredential } from "@azure/identity"
+
+const { account = "myblobs", container = "myimages" } = env.vars
+const blobServiceClient = new BlobServiceClient(
+    `https://${account}.blob.core.windows.net`,
+    new DefaultAzureCredential()
+)
+const containerClient = blobServiceClient.getContainerClient(container)
+```
+
+If you do not have a specific blob in mind, you can iterate through the blobs,
+and download them into a buffer (`buf`).
+
+```ts "image"
+import { buffer } from "node:stream/consumers"
+
+for await (const blob of containerClient.listBlobsFlat()) {
+    const blockBlobClient = containerClient.getBlockBlobClient(blob.name)
+    const downloadBlockBlobResponse = await blockBlobClient.download(0)
+    const body = await downloadBlockBlobResponse.readableStreamBody
+    const image = await buffer(body)
+    ...
+```
+
+## Using images in the prompt
+
+The `image` buffer can be passed in `defImages` to be used in the prompt.
+
+```ts
+    defImages(image, { detail: "low" })
+```
+
+However since images can be "heavy", you will most likely have to use
+[inline prompts](/genaiscript/reference/prompts/inline-prompts) to split into smaller queries. (Note the use of `_.`)
+
+```ts 'await runPrompt(_ =>' '_.'
+for await (const blob of containerClient.listBlobsFlat()) {
+    ...
+    const res = await runPrompt(_ => {
+        _.defImages(image, { detail: "low" })
+        _.$`Describe the image.`
+    })
+    // res contains the LLM response for the inner prompt
+    ...
+```
+
+## Summarizing results
+
+To summarize all images, we store each image summary using the `def` function and 
+add prompting to summarize the descriptions.
+
+```ts
+    ...
+    def("IMAGES_SUMMARY", { filename: blob.name, content: res.text })
+}
+$`Summarize IMAGES_SUMMARY.`
+```
+
+## Full source
+
+<Code code={source} wrap={true} lang="js" title="azure-blobs.genai.mts" />
+
diff --git a/docs/src/content/docs/guides/llm-as-tool.mdx b/docs/src/content/docs/guides/llm-as-tool.mdx
@@ -1,7 +1,10 @@
 ---
 title: LLM as a tool
 sidebar:
-    order: 60
+  order: 60
+description: Create tools and inline prompts using LLM models for executing various tasks
+keywords: LLM, inline prompts, tools, gpt-3.5-turbo, task automation
+
 ---
 
 It is possible [tools](/genaiscript/reference/scripts/tools)

diff --git a/docs/src/content/docs/guides/transformers-js.mdx b/docs/src/content/docs/guides/transformers-js.mdx
@@ -1,7 +1,12 @@
 ---
 title: Transformer.js
 sidebar:
-    order: 20
+  order: 20
+description: Implement summarization with Transformers.js and leverage local
+  hardware acceleration
+keywords: Transformers.js, summarization, onnxruntime, CPU/GPU acceleration,
+  pretrained models
+
 ---
 import { Code } from "@astrojs/starlight/components"
 import sampleSrc from "../../../../../packages/sample/genaisrc/summary-with-transformers.genai?raw"

diff --git a/docs/src/content/docs/guides/using-secrets.mdx b/docs/src/content/docs/guides/using-secrets.mdx
@@ -2,6 +2,9 @@
 title: Using Secrets
 sidebar:
   order: 20
+description: Utilize secrets to augment documents with TypeScript and REST APIs
+keywords: TypeScript, secrets, REST API, document augmentation, web search
+
 ---
 import { Code } from '@astrojs/starlight/components';
 import tavilyCode from "../../../../../packages/sample/genaisrc/tavily.mjs?raw"

diff --git a/docs/src/content/docs/guides/zod-schema.mdx b/docs/src/content/docs/guides/zod-schema.mdx
@@ -2,6 +2,9 @@
 title: Zod Schema
 sidebar:
   order: 80
+description: Learn how to define and convert TypeScript-first Zod schemas to JSON schema
+keywords: TypeScript, Zod schema, JSON schema, schema validation, type inference
+
 ---
 
 [zod](https://zod.dev/) is a TypeScript-first schema validation with static type inference. 

diff --git a/docs/src/content/docs/reference/scripts/chat-participants.mdx b/docs/src/content/docs/reference/scripts/chat-participants.mdx
@@ -1,7 +1,11 @@
 ---
 title: Chat Participants
 sidebar:
-    order: 50
+  order: 50
+description: Create multi-turn chats or simulate conversations with multiple
+  chat participants
+keywords: chat, multi-turn, conversation, chat participant, engagement
+
 ---
 import { Code } from '@astrojs/starlight/components';
 import scriptSource from "../../../../../../packages/sample/genaisrc/multi-turn.genai.js?raw"

diff --git a/docs/src/content/docs/reference/scripts/images.md b/docs/src/content/docs/reference/scripts/images.md
@@ -15,3 +15,29 @@ defImages(env.files)
 ```
 
 Read more about [OpenAI Vision](https://platform.openai.com/docs/guides/vision/limitations).
+
+## URLs
+
+Public URLs (that do not require authentication) will be passed directly to OpenAI.
+
+```js
+defImages(
+    "https://github.com/microsoft/genaiscript/blob/main/docs/public/images/logo.png?raw=true"
+)
+```
+
+Local files are loaded and encoded as a data uri.
+
+## Buffer, Blob
+
+The `defImages` function also supports [Buffer](https://nodejs.org/api/buffer.html)
+and [Blob](https://developer.mozilla.org/en-US/docs/Web/API/Blob).
+
+
+This example takes a screenshot of bing.com and adds it to the images.
+
+```js
+const page = await host.browse("https://bing.com")
+const screenshot = await page.screenshot() // returns a node.js Buffer
+defImages(screenshot)
+```
diff --git a/docs/src/content/docs/reference/scripts/prompty.md b/docs/src/content/docs/reference/scripts/prompty.md
@@ -1,7 +1,11 @@
 ---
 title: Prompty
 sidebar:
-    order: 51
+  order: 51
+description: Learn about the .prompty file format for parameterized prompts and
+  its integration with GenAIScript for AI scripting.
+keywords: prompty, scripts, AI, parameterized prompts, automation
+
 ---
 
 GenAIScript supports running [.prompty](https://prompty.ai/) files as scripts (with some limitations).

diff --git a/genaisrc/genaiscript.d.ts b/genaisrc/genaiscript.d.ts
diff --git a/packages/core/src/genaisrc/genaiscript.d.ts b/packages/core/src/genaisrc/genaiscript.d.ts