Add basic support for Google AI models

docs/src/content/docs/getting-started/configuration.mdx

@@ -20,7 +20,6 @@ import lmSelectAlt from "../../../assets/vscode-language-models-select.png.txt?r
 import oaiModelsSrc from "../../../assets/openai-model-names.png"
 import oaiModelsAlt from "../../../assets/openai-model-names.png.txt?raw"
 
-
 You will need to configure the LLM connection and authorization secrets.
 
 :::tip
@@ -148,26 +147,25 @@ envFile: ~/.env.genaiscript
 
 ### No .env file
 
-If you do not want to use a `.env` file, make sure to populate the environment variables 
+If you do not want to use a `.env` file, make sure to populate the environment variables
 of the genaiscript process with the configuration values.
 
 Here are some common examples:
 
-- Using bash syntax
+-   Using bash syntax
 
 ```sh
 OPENAI_API_KEY="value" npx --yes genaiscript run ...
 ```
 
-- GitHub Action configuration
+-   GitHub Action configuration
 
 ```yaml title=".github/workflows/genaiscript.yml"
-    run: npx --yes genaiscript run ...
-    env:
-        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+run: npx --yes genaiscript run ...
+env:
+    OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
 ```
 
-
 ## OpenAI
 
 This provider, `openai`, is the OpenAI chat model provider.
@@ -685,6 +683,62 @@ model3=key3
 "
 ```
 
+## Google AI <a href="" id="google" />
+
+The `google` provider allows you to use Google AI models. It gives you access
+
+:::note
+
+GenAIScript uses the [OpenAI compatibility](https://ai.google.dev/gemini-api/docs/openai) layer of Google AI,
+so some [limitations](https://ai.google.dev/gemini-api/docs/openai#current-limitations) apply.
+
+:::
+
+<Steps>
+
+<ol>
+
+<li>
+
+Open [Google AI Studio](https://aistudio.google.com/app/apikey) and create a new API key.
+
+</li>
+
+<li>
+
+Update the `.env` file with the API key.
+
+```txt title=".env"
+GOOGLE_API_KEY=...
+```
+
+</li>
+
+<li>
+
+Find the model identifier in the [Gemini documentation](https://ai.google.dev/gemini-api/docs/models/gemini)
+and use it in your script or cli with the `google` provider.
+
+```py "gemini-1.5-pro-002"
+...
+const model = genAI.getGenerativeModel({
+  model: "gemini-1.5-pro-002",
+});
+...
+```
+
+then use the model identifier in your script.
+
+```js "gemini-1.5-pro-002"
+script({ model: "google:gemini-1.5-pro-002" })
+```
+
+</li>
+
+</ol>
+
+</Steps>
+
 ## GitHub Copilot Chat Models <a id="github-copilot" href=""></a>
 
 If you have access to **GitHub Copilot Chat in Visual Studio Code**,

docs/src/content/docs/reference/scripts/system.mdx

@@ -3401,6 +3401,7 @@ defTool(
             },
             {
                 model: "vision",
+                cache: "vision_ask_image",
                 system: [
                     "system",
                     "system.assistant",

packages/core/src/chat.ts

@@ -818,7 +818,7 @@
         topLogprobs,
     } = genOptions
     const top_logprobs = genOptions.topLogprobs > 0 ? topLogprobs : undefined
-    const logprobs = genOptions.logprobs || top_logprobs > 0
+    const logprobs = genOptions.logprobs || top_logprobs > 0 ? true : undefined
     traceLanguageModelConnection(trace, genOptions, connectionToken)
     const tools: ChatCompletionTool[] = toolDefinitions?.length
         ? toolDefinitions.map(

packages/core/src/connection.ts

@@ -23,6 +23,8 @@
     HUGGINGFACE_API_BASE,
     OLLAMA_API_BASE,
     OLLAMA_DEFAUT_PORT,
+    MODEL_PROVIDER_GOOGLE,
+    GOOGLE_API_BASE,
 } from "./constants"
 import { fileExists, readText, writeText } from "./fs"
 import {
@@ -129,7 +131,7 @@
             token,
             source: "env: OPENAI_API_...",
             version,
-        }
+        } satisfies LanguageModelConfiguration
     }
 
     if (provider === MODEL_PROVIDER_GITHUB) {
@@ -148,7 +150,7 @@
             type,
             token,
             source: `env: ${tokenVar}`,
-        }
+        } satisfies LanguageModelConfiguration
     }
 
     if (provider === MODEL_PROVIDER_AZURE_OPENAI) {
@@ -194,7 +196,7 @@
                 : "env: AZURE_OPENAI_API_... + Entra ID",
             version,
             azureCredentialsType,
-        }
+        } satisfies LanguageModelConfiguration
     }
 
     if (provider === MODEL_PROVIDER_AZURE_SERVERLESS_OPENAI) {
@@ -239,7 +241,7 @@
                 : "env: AZURE_SERVERLESS_OPENAI_API_... + Entra ID",
             version,
             azureCredentialsType,
-        }
+        } satisfies LanguageModelConfiguration
     }
 
     if (provider === MODEL_PROVIDER_AZURE_SERVERLESS_MODELS) {
@@ -281,7 +283,25 @@
                 ? "env: AZURE_SERVERLESS_MODELS_API_..."
                 : "env: AZURE_SERVERLESS_MODELS_API_... + Entra ID",
             version,
-        }
+        } satisfies LanguageModelConfiguration
+    }
+
+    if (provider === MODEL_PROVIDER_GOOGLE) {
+        const token = env.GOOGLE_API_KEY
+        if (!token) return undefined
+        if (token === PLACEHOLDER_API_KEY)
+            throw new Error("GOOGLE_API_KEY not configured")
+        const base = env.GOOGLE_API_BASE || GOOGLE_API_BASE
+        if (base === PLACEHOLDER_API_BASE)
+            throw new Error("GOOGLE_API_BASE not configured")
+        return {
+            provider,
+            model,
+            base,
+            token,
+            type: "openai",
+            source: "env: GOOGLE_API_...",
+        } satisfies LanguageModelConfiguration
     }
 
     if (provider === MODEL_PROVIDER_ANTHROPIC) {
@@ -301,7 +321,7 @@
             base,
             version,
             source,
-        }
+        } satisfies LanguageModelConfiguration
     }
 
     if (provider === MODEL_PROVIDER_OLLAMA) {
@@ -314,7 +334,7 @@
             token: "ollama",
             type: "openai",
             source: "env: OLLAMA_HOST",
-        }
+        } satisfies LanguageModelConfiguration
     }
 
     if (provider === MODEL_PROVIDER_HUGGINGFACE) {

packages/core/src/constants.ts

@@ -69,6 +69,7 @@ export const DEFAULT_VISION_MODEL_CANDIDATES = [
     "azure_serverless:gpt-4o",
     DEFAULT_MODEL,
     "anthropic:claude-2",
+    "google:gemini-1.5-pro-002",
     "github:gpt-4o",
 ]
 export const DEFAULT_SMALL_MODEL = "openai:gpt-4o-mini"
@@ -78,6 +79,7 @@ export const DEFAULT_SMALL_MODEL_CANDIDATES = [
     DEFAULT_SMALL_MODEL,
     "anthropic:claude-instant-1",
     "github:gpt-4o-mini",
+    "google:gemini-1.5-flash-002",
     "client:gpt-4-mini",
 ]
 export const DEFAULT_EMBEDDINGS_MODEL_CANDIDATES = [
@@ -160,6 +162,7 @@ export const EMOJI_UNDEFINED = "?"
 export const MODEL_PROVIDER_OPENAI = "openai"
 export const MODEL_PROVIDER_GITHUB = "github"
 export const MODEL_PROVIDER_AZURE_OPENAI = "azure"
+export const MODEL_PROVIDER_GOOGLE = "google"
 export const MODEL_PROVIDER_AZURE_SERVERLESS_OPENAI = "azure_serverless"
 export const MODEL_PROVIDER_AZURE_SERVERLESS_MODELS = "azure_serverless_models"
 export const MODEL_PROVIDER_OLLAMA = "ollama"
@@ -203,6 +206,8 @@ export const DOCS_CONFIGURATION_AICI_URL =
     "https://microsoft.github.io/genaiscript/reference/scripts/aici/"
 export const DOCS_CONFIGURATION_ANTHROPIC_URL =
     "https://microsoft.github.io/genaiscript/getting-started/configuration/#anthropic"
+export const DOCS_CONFIGURATION_GOOGLE_URL =
+    "https://microsoft.github.io/genaiscript/getting-started/configuration/#google"
 export const DOCS_CONFIGURATION_HUGGINGFACE_URL =
     "https://microsoft.github.io/genaiscript/getting-started/configuration/#huggingface"
 export const DOCS_CONFIGURATION_CONTENT_SAFETY_URL =
@@ -247,6 +252,11 @@ export const MODEL_PROVIDERS = Object.freeze([
         detail: "Anthropic models",
         url: DOCS_CONFIGURATION_ANTHROPIC_URL,
     },
+    {
+        id: MODEL_PROVIDER_GOOGLE,
+        detail: "Google AI",
+        url: DOCS_CONFIGURATION_GOOGLE_URL,
+    },
     {
         id: MODEL_PROVIDER_HUGGINGFACE,
         detail: "Hugging Face models",
@@ -357,3 +367,6 @@ export const CHOICE_LOGIT_BIAS = 5
 
 export const SANITIZED_PROMPT_INJECTION =
     "...prompt injection detected, content removed..."
+
+export const GOOGLE_API_BASE =
+    "https://generativelanguage.googleapis.com/v1beta/openai/"

packages/core/src/fetch.ts

@@ -163,12 +163,12 @@
                         ? "Bearer ***" // Mask Bearer tokens
                         : "***") // Mask other authorization headers
             )
-    const cmd = `curl ${url} \\
+    const cmd = `curl "${url}" \\
+--no-buffer \\
 ${Object.entries(headers)
     .map(([k, v]) => `-H "${k}: ${v}"`)
-    .join("\\\n")} \\
+    .join(" \\\n")} \\
 -d '${JSON.stringify(body, null, 2).replace(/'/g, "'\\''")}'
---no-buffer
 `
     if (trace) trace.detailsFenced(`✉️ fetch`, cmd, "bash")
     else logVerbose(cmd)

packages/core/src/genaisrc/system.vision_ask_image.genai.mjs

@@ -40,6 +40,7 @@ defTool(
             },
             {
                 model: "vision",
+                cache: "vision_ask_image",
                 system: [
                     "system",
                     "system.assistant",

packages/core/src/ollama.ts

@@ -77,7 +77,7 @@ async function listModels(
     cfg: LanguageModelConfiguration
 ): Promise<LanguageModelInfo[]> {
     // Create a fetch instance to make HTTP requests
-    const fetch = await createFetch()
+    const fetch = await createFetch({ retries: 1 })
     // Fetch the list of models from the remote API
     const res = await fetch(cfg.base.replace("/v1", "/api/tags"), {
         method: "GET",

packages/core/src/openai.ts

@@ -203,8 +203,8 @@ export const OpenAIChatCompletion: ChatCompletionHandler = async (
     trace.dispatchChange()
 
     const fetchHeaders: HeadersInit = {
-        ...getConfigHeaders(cfg),
         "Content-Type": "application/json",
+        ...getConfigHeaders(cfg),
         ...(headers || {}),
     }
     traceFetchPost(trace, url, fetchHeaders as any, postReq)
@@ -282,7 +282,9 @@ export const OpenAIChatCompletion: ChatCompletionHandler = async (
                 numTokens += estimateTokens(delta.content, encoder)
                 chatResp += delta.content
                 tokens.push(
-                    ...serializeChunkChoiceToLogProbs(choice as ChatCompletionChunkChoice)
+                    ...serializeChunkChoiceToLogProbs(
+                        choice as ChatCompletionChunkChoice
+                    )
                 )
                 trace.appendToken(delta.content)
             } else if (Array.isArray(delta.tool_calls)) {
@@ -409,7 +411,7 @@ export const OpenAIChatCompletion: ChatCompletionHandler = async (
 async function listModels(
     cfg: LanguageModelConfiguration
 ): Promise<LanguageModelInfo[]> {
-    const fetch = await createFetch()
+    const fetch = await createFetch({ retries: 1 })
     const res = await fetch(cfg.base + "/models", {
         method: "GET",
         headers: {

packages/core/src/pricing.json

@@ -270,5 +270,50 @@
     "azure_serverless_models:ministral-3b": {
         "price_per_million_input_tokens": 0.04,
         "price_per_million_output_tokens": 0.04
+    },
+    "google:gemini-1.5-flash": {
+        "price_per_million_input_tokens": 0.075,
+        "price_per_million_output_tokens": 0.3
+    },
+    "google:gemini-1.5-flash-002": {
+        "price_per_million_input_tokens": 0.075,
+        "price_per_million_output_tokens": 0.3
+    },
+    "google:gemini-1.5-flash-8b": {
+        "price_per_million_input_tokens": 0.0375,
+        "price_per_million_output_tokens": 0.15,
+        "tiers": [
+            {
+                "context_size": 128000,
+                "price_per_million_input_tokens": 0.075,
+                "price_per_million_output_tokens": 0.3
+            }
+        ]
+    },
+    "google:gemini-1.5-pro": {
+        "price_per_million_input_tokens": 1.25,
+        "price_per_million_output_tokens": 5,
+        "tiers": [
+            {
+                "context_size": 128000,
+                "price_per_million_input_tokens": 2.5,
+                "price_per_million_output_tokens": 10
+            }
+        ]
+    },
+    "google:gemini-1.5-pro-002": {
+        "price_per_million_input_tokens": 1.25,
+        "price_per_million_output_tokens": 5,
+        "tiers": [
+            {
+                "context_size": 128000,
+                "price_per_million_input_tokens": 2.5,
+                "price_per_million_output_tokens": 10
+            }
+        ]
+    },
+    "google:gemini-1-pro": {
+        "price_per_million_input_tokens": 0.5,
+        "price_per_million_output_tokens": 1.5
     }
 }

packages/core/src/tools.ts

@@ -2,6 +2,7 @@ import {
     MODEL_PROVIDER_AZURE_OPENAI,
     MODEL_PROVIDER_AZURE_SERVERLESS_MODELS,
     MODEL_PROVIDER_GITHUB,
+    MODEL_PROVIDER_GOOGLE,
     MODEL_PROVIDER_OLLAMA,
     MODEL_PROVIDER_OPENAI,
 } from "./constants"
@@ -38,6 +39,9 @@ export function isToolsSupported(modelId: string): boolean | undefined {
         [MODEL_PROVIDER_OPENAI]: oai,
         [MODEL_PROVIDER_AZURE_OPENAI]: oai,
         [MODEL_PROVIDER_AZURE_SERVERLESS_MODELS]: oai,
+        [MODEL_PROVIDER_GOOGLE]: {
+            // all supported
+        },
         [MODEL_PROVIDER_GITHUB]: {
             "Phi-3.5-mini-instruct": false,
         },