diff --git a/docs/src/content/docs/reference/scripts/images.md b/docs/src/content/docs/reference/scripts/images.md index 5dcc07f9f8..abe7bae6bc 100644 --- a/docs/src/content/docs/reference/scripts/images.md +++ b/docs/src/content/docs/reference/scripts/images.md @@ -15,3 +15,29 @@ defImages(env.files) ``` Read more about [OpenAI Vision](https://platform.openai.com/docs/guides/vision/limitations). + +## URLs + +Public URLs (that do not require authentication) will be passed directly to OpenAI. + +```js +defImages( + "https://github.com/microsoft/genaiscript/blob/main/docs/public/images/logo.png?raw=true" +) +``` + +Local files are loaded and encoded as a data uri. + +## Buffer, Blob + +The `defImage` function also supports [Buffer](https://nodejs.org/api/buffer.html) +and [Blob](https://developer.mozilla.org/en-US/docs/Web/API/Blob). + + +This example takes a screenshot of bing.com and adds it to the images. + +```js +const page = await host.browse("https://bing.com") +const screenshot = await page.screenshot() // returns a node.js Buffer +defImages(screenshot) +``` diff --git a/packages/sample/genaisrc/azure-blog-storage.genai.mts b/packages/sample/genaisrc/azure-blog-storage.genai.mts new file mode 100644 index 0000000000..b313b5d8cc --- /dev/null +++ b/packages/sample/genaisrc/azure-blog-storage.genai.mts @@ -0,0 +1,35 @@ +import { BlobServiceClient } from "@azure/storage-blob" +import { DefaultAzureCredential } from "@azure/identity" + +script({ + parameters: { + account: { + description: "Azure Storage Account Name", + default: "myaccount", + type: "string", + }, + container: { + description: "Azure Storage Container Name", + default: "mycontainer", + type: "string", + }, + }, +}) + +const { account, container } = env.vars +const blobServiceClient = new BlobServiceClient( + `https://${account}.blob.core.windows.net`, + new DefaultAzureCredential() +) +const containerClient = blobServiceClient.getContainerClient(container) +for await (const blob of containerClient.listBlobsFlat()) { + const blockBlobClient = containerClient.getBlockBlobClient(blob.name) + const downloadBlockBlobResponse = await blockBlobClient.download(0) + const body = await downloadBlockBlobResponse.readableStreamBody + const buffer = await body.read() + const res = await runPrompt((_) => { + _.defImages(buffer) + _.$`Describe the image.` + }) + // do something with res? +} diff --git a/packages/sample/package.json b/packages/sample/package.json index 1da0f8b35d..3764bd0534 100644 --- a/packages/sample/package.json +++ b/packages/sample/package.json @@ -18,6 +18,8 @@ "@agentic/calculator": "^7.0.0", "@agentic/core": "^7.0.0", "@agentic/weather": "^7.0.0", + "@azure/identity": "^4.4.1", + "@azure/storage-blob": "^12.24.0", "@tidyjs/tidy": "^2.5.2", "@xenova/transformers": "^2.17.2", "vectorstore": "^0.0.4", diff --git a/yarn.lock b/yarn.lock index 10bb102296..c7a6869093 100644 --- a/yarn.lock +++ b/yarn.lock @@ -60,7 +60,7 @@ "@azure/core-util" "^1.1.0" tslib "^2.6.2" -"@azure/core-client@^1.9.2": +"@azure/core-client@^1.3.0", "@azure/core-client@^1.6.2", "@azure/core-client@^1.9.2": version "1.9.2" resolved "https://registry.yarnpkg.com/@azure/core-client/-/core-client-1.9.2.tgz#6fc69cee2816883ab6c5cdd653ee4f2ff9774f74" integrity sha512-kRdry/rav3fUKHl/aDLd/pDLcB+4pOFwPPTVEExuMyaI5r+JBbMWqRbCY1pn5BniDaU3lRxO9eaQ1AmSMehl/w== @@ -73,7 +73,33 @@ "@azure/logger" "^1.0.0" tslib "^2.6.2" -"@azure/core-rest-pipeline@^1.1.0", "@azure/core-rest-pipeline@^1.9.1": +"@azure/core-http-compat@^2.0.0": + version "2.1.2" + resolved "https://registry.yarnpkg.com/@azure/core-http-compat/-/core-http-compat-2.1.2.tgz#d1585ada24ba750dc161d816169b33b35f762f0d" + integrity sha512-5MnV1yqzZwgNLLjlizsU3QqOeQChkIXw781Fwh1xdAqJR5AA32IUaq6xv1BICJvfbHoa+JYcaij2HFkhLbNTJQ== + dependencies: + "@azure/abort-controller" "^2.0.0" + "@azure/core-client" "^1.3.0" + "@azure/core-rest-pipeline" "^1.3.0" + +"@azure/core-lro@^2.2.0": + version "2.7.2" + resolved "https://registry.yarnpkg.com/@azure/core-lro/-/core-lro-2.7.2.tgz#787105027a20e45c77651a98b01a4d3b01b75a08" + integrity sha512-0YIpccoX8m/k00O7mDDMdJpbr6mf1yWo2dfmxt5A8XVZVVMz2SSKaEbMCeJRvgQ0IaSlqhjT47p4hVIRRy90xw== + dependencies: + "@azure/abort-controller" "^2.0.0" + "@azure/core-util" "^1.2.0" + "@azure/logger" "^1.0.0" + tslib "^2.6.2" + +"@azure/core-paging@^1.1.1": + version "1.6.2" + resolved "https://registry.yarnpkg.com/@azure/core-paging/-/core-paging-1.6.2.tgz#40d3860dc2df7f291d66350b2cfd9171526433e7" + integrity sha512-YKWi9YuCU04B55h25cnOYZHxXYtEvQEbKST5vqRga7hWY9ydd3FZHdeQF8pyh+acWZvppw13M/LMGx0LABUVMA== + dependencies: + tslib "^2.6.2" + +"@azure/core-rest-pipeline@^1.1.0", "@azure/core-rest-pipeline@^1.10.1", "@azure/core-rest-pipeline@^1.3.0", "@azure/core-rest-pipeline@^1.9.1": version "1.16.3" resolved "https://registry.yarnpkg.com/@azure/core-rest-pipeline/-/core-rest-pipeline-1.16.3.tgz#bde3bc3ebad7f885ddd9de6af5e5a8fc254b287e" integrity sha512-VxLk4AHLyqcHsfKe4MZ6IQ+D+ShuByy+RfStKfSjxJoL3WBWq17VNmrz8aT8etKzqc2nAeIyLxScjpzsS4fz8w== @@ -87,14 +113,14 @@ https-proxy-agent "^7.0.0" tslib "^2.6.2" -"@azure/core-tracing@^1.0.0", "@azure/core-tracing@^1.0.1": +"@azure/core-tracing@^1.0.0", "@azure/core-tracing@^1.0.1", "@azure/core-tracing@^1.1.2": version "1.1.2" resolved "https://registry.yarnpkg.com/@azure/core-tracing/-/core-tracing-1.1.2.tgz#065dab4e093fb61899988a1cdbc827d9ad90b4ee" integrity sha512-dawW9ifvWAWmUm9/h+/UQ2jrdvjCJ7VJEuCJ6XVNudzcOwm53BFZH4Q845vjfgoUAM8ZxokvVNxNxAITc502YA== dependencies: tslib "^2.6.2" -"@azure/core-util@^1.1.0", "@azure/core-util@^1.3.0", "@azure/core-util@^1.6.1", "@azure/core-util@^1.9.0": +"@azure/core-util@^1.1.0", "@azure/core-util@^1.2.0", "@azure/core-util@^1.3.0", "@azure/core-util@^1.6.1", "@azure/core-util@^1.9.0": version "1.9.2" resolved "https://registry.yarnpkg.com/@azure/core-util/-/core-util-1.9.2.tgz#1dc37dc5b0dae34c578be62cf98905ba7c0cafe7" integrity sha512-l1Qrqhi4x1aekkV+OlcqsJa4AnAkj5p0JV8omgwjaV9OAbP41lvrMvs+CptfetKkeEaGRGSzby7sjPZEX7+kkQ== @@ -102,6 +128,14 @@ "@azure/abort-controller" "^2.0.0" tslib "^2.6.2" +"@azure/core-xml@^1.3.2": + version "1.4.3" + resolved "https://registry.yarnpkg.com/@azure/core-xml/-/core-xml-1.4.3.tgz#a74f37a0e584fee7e9adae19f51016d4b59e9ca2" + integrity sha512-D6G7FEmDiTctPKuWegX2WTrS1enKZwqYwdKTO6ZN6JMigcCehlT0/CYl+zWpI9vQ9frwwp7GQT3/owaEXgnOsA== + dependencies: + fast-xml-parser "^4.3.2" + tslib "^2.6.2" + "@azure/identity@^4.1.0", "@azure/identity@^4.4.1": version "4.4.1" resolved "https://registry.yarnpkg.com/@azure/identity/-/identity-4.4.1.tgz#490fa2ad26786229afa36411892bb53dfa3478d3" @@ -150,6 +184,25 @@ jsonwebtoken "^9.0.0" uuid "^8.3.0" +"@azure/storage-blob@^12.24.0": + version "12.24.0" + resolved "https://registry.yarnpkg.com/@azure/storage-blob/-/storage-blob-12.24.0.tgz#d4ae1e29574b4a19d90eaf082cfde95f996d3f9b" + integrity sha512-l8cmWM4C7RoNCBOImoFMxhTXe1Lr+8uQ/IgnhRNMpfoA9bAFWoLG4XrWm6O5rKXortreVQuD+fc1hbzWklOZbw== + dependencies: + "@azure/abort-controller" "^1.0.0" + "@azure/core-auth" "^1.4.0" + "@azure/core-client" "^1.6.2" + "@azure/core-http-compat" "^2.0.0" + "@azure/core-lro" "^2.2.0" + "@azure/core-paging" "^1.1.1" + "@azure/core-rest-pipeline" "^1.10.1" + "@azure/core-tracing" "^1.1.2" + "@azure/core-util" "^1.6.1" + "@azure/core-xml" "^1.3.2" + "@azure/logger" "^1.0.0" + events "^3.0.0" + tslib "^2.2.0" + "@babel/code-frame@^7.0.0": version "7.24.7" resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.24.7.tgz#882fd9e09e8ee324e496bd040401c6f046ef4465" @@ -2227,7 +2280,7 @@ fast-uri@^3.0.1: resolved "https://registry.yarnpkg.com/fast-uri/-/fast-uri-3.0.1.tgz#cddd2eecfc83a71c1be2cc2ef2061331be8a7134" integrity sha512-MWipKbbYiYI0UC7cl8m/i/IWTqfC8YXsqjzybjddLsFjStroQzsHXkc73JutMvBiXmOvapk+axIl79ig5t55Bw== -fast-xml-parser@^4.5.0: +fast-xml-parser@^4.3.2, fast-xml-parser@^4.5.0: version "4.5.0" resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-4.5.0.tgz#2882b7d01a6825dfdf909638f2de0256351def37" integrity sha512-/PlTQCI96+fZMAOLMZK4CWG1ItCbfZ/0jx7UIJFChPNrx7tcEgerUgWbeieCM9MfHInUDyK8DWYZ+YrywDJuTg==