diff --git a/docs/src/content/docs/getting-started/configuration.mdx b/docs/src/content/docs/getting-started/configuration.mdx index 7c0846b619..314e7d7af5 100644 --- a/docs/src/content/docs/getting-started/configuration.mdx +++ b/docs/src/content/docs/getting-started/configuration.mdx @@ -20,7 +20,6 @@ import lmSelectAlt from "../../../assets/vscode-language-models-select.png.txt?r import oaiModelsSrc from "../../../assets/openai-model-names.png" import oaiModelsAlt from "../../../assets/openai-model-names.png.txt?raw" - You will need to configure the LLM connection and authorization secrets. :::tip @@ -148,26 +147,25 @@ envFile: ~/.env.genaiscript ### No .env file -If you do not want to use a `.env` file, make sure to populate the environment variables +If you do not want to use a `.env` file, make sure to populate the environment variables of the genaiscript process with the configuration values. Here are some common examples: -- Using bash syntax +- Using bash syntax ```sh OPENAI_API_KEY="value" npx --yes genaiscript run ... ``` -- GitHub Action configuration +- GitHub Action configuration ```yaml title=".github/workflows/genaiscript.yml" - run: npx --yes genaiscript run ... - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} +run: npx --yes genaiscript run ... +env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} ``` - ## OpenAI This provider, `openai`, is the OpenAI chat model provider. @@ -685,6 +683,62 @@ model3=key3 " ``` +## Google AI + +The `google` provider allows you to use Google AI models. It gives you access + +:::note + +GenAIScript uses the [OpenAI compatibility](https://ai.google.dev/gemini-api/docs/openai) layer of Google AI, +so some [limitations](https://ai.google.dev/gemini-api/docs/openai#current-limitations) apply. + +::: + + + +
    + +
  1. + +Open [Google AI Studio](https://aistudio.google.com/app/apikey) and create a new API key. + +
    2. + +
  2. + +Update the `.env` file with the API key. + +```txt title=".env" +GOOGLE_API_KEY=... +``` + +
    3. + +
  3. + +Find the model identifier in the [Gemini documentation](https://ai.google.dev/gemini-api/docs/models/gemini) +and use it in your script or cli with the `google` provider. + +```py "gemini-1.5-pro-002" +... +const model = genAI.getGenerativeModel({ + model: "gemini-1.5-pro-002", +}); +... +``` + +then use the model identifier in your script. + +```js "gemini-1.5-pro-002" +script({ model: "google:gemini-1.5-pro-002" }) +``` + +
    4. + +
+ + + ## GitHub Copilot Chat Models If you have access to **GitHub Copilot Chat in Visual Studio Code**, diff --git a/docs/src/content/docs/reference/scripts/system.mdx b/docs/src/content/docs/reference/scripts/system.mdx index 9c9df5448e..6a184b100e 100644 --- a/docs/src/content/docs/reference/scripts/system.mdx +++ b/docs/src/content/docs/reference/scripts/system.mdx @@ -3401,6 +3401,7 @@ defTool( }, { model: "vision", + cache: "vision_ask_image", system: [ "system", "system.assistant", diff --git a/packages/core/src/chat.ts b/packages/core/src/chat.ts index 016a03bd5f..b0877fb6c5 100644 --- a/packages/core/src/chat.ts +++ b/packages/core/src/chat.ts @@ -818,7 +818,7 @@ export async function executeChatSession( topLogprobs, } = genOptions const top_logprobs = genOptions.topLogprobs > 0 ? topLogprobs : undefined - const logprobs = genOptions.logprobs || top_logprobs > 0 + const logprobs = genOptions.logprobs || top_logprobs > 0 ? true : undefined traceLanguageModelConnection(trace, genOptions, connectionToken) const tools: ChatCompletionTool[] = toolDefinitions?.length ? toolDefinitions.map( diff --git a/packages/core/src/connection.ts b/packages/core/src/connection.ts index 7b5e56c444..dbdb585341 100644 --- a/packages/core/src/connection.ts +++ b/packages/core/src/connection.ts @@ -23,6 +23,8 @@ import { HUGGINGFACE_API_BASE, OLLAMA_API_BASE, OLLAMA_DEFAUT_PORT, + MODEL_PROVIDER_GOOGLE, + GOOGLE_API_BASE, } from "./constants" import { fileExists, readText, writeText } from "./fs" import { @@ -129,7 +131,7 @@ export async function parseTokenFromEnv( token, source: "env: OPENAI_API_...", version, - } + } satisfies LanguageModelConfiguration } if (provider === MODEL_PROVIDER_GITHUB) { @@ -148,7 +150,7 @@ export async function parseTokenFromEnv( type, token, source: `env: ${tokenVar}`, - } + } satisfies LanguageModelConfiguration } if (provider === MODEL_PROVIDER_AZURE_OPENAI) { @@ -194,7 +196,7 @@ export async function parseTokenFromEnv( : "env: AZURE_OPENAI_API_... + Entra ID", version, azureCredentialsType, - } + } satisfies LanguageModelConfiguration } if (provider === MODEL_PROVIDER_AZURE_SERVERLESS_OPENAI) { @@ -239,7 +241,7 @@ export async function parseTokenFromEnv( : "env: AZURE_SERVERLESS_OPENAI_API_... + Entra ID", version, azureCredentialsType, - } + } satisfies LanguageModelConfiguration } if (provider === MODEL_PROVIDER_AZURE_SERVERLESS_MODELS) { @@ -281,7 +283,25 @@ export async function parseTokenFromEnv( ? "env: AZURE_SERVERLESS_MODELS_API_..." : "env: AZURE_SERVERLESS_MODELS_API_... + Entra ID", version, - } + } satisfies LanguageModelConfiguration + } + + if (provider === MODEL_PROVIDER_GOOGLE) { + const token = env.GOOGLE_API_KEY + if (!token) return undefined + if (token === PLACEHOLDER_API_KEY) + throw new Error("GOOGLE_API_KEY not configured") + const base = env.GOOGLE_API_BASE || GOOGLE_API_BASE + if (base === PLACEHOLDER_API_BASE) + throw new Error("GOOGLE_API_BASE not configured") + return { + provider, + model, + base, + token, + type: "openai", + source: "env: GOOGLE_API_...", + } satisfies LanguageModelConfiguration } if (provider === MODEL_PROVIDER_ANTHROPIC) { @@ -301,7 +321,7 @@ export async function parseTokenFromEnv( base, version, source, - } + } satisfies LanguageModelConfiguration } if (provider === MODEL_PROVIDER_OLLAMA) { @@ -314,7 +334,7 @@ export async function parseTokenFromEnv( token: "ollama", type: "openai", source: "env: OLLAMA_HOST", - } + } satisfies LanguageModelConfiguration } if (provider === MODEL_PROVIDER_HUGGINGFACE) { diff --git a/packages/core/src/constants.ts b/packages/core/src/constants.ts index bd3ddf842c..3189857ef4 100644 --- a/packages/core/src/constants.ts +++ b/packages/core/src/constants.ts @@ -69,6 +69,7 @@ export const DEFAULT_VISION_MODEL_CANDIDATES = [ "azure_serverless:gpt-4o", DEFAULT_MODEL, "anthropic:claude-2", + "google:gemini-1.5-pro-002", "github:gpt-4o", ] export const DEFAULT_SMALL_MODEL = "openai:gpt-4o-mini" @@ -78,6 +79,7 @@ export const DEFAULT_SMALL_MODEL_CANDIDATES = [ DEFAULT_SMALL_MODEL, "anthropic:claude-instant-1", "github:gpt-4o-mini", + "google:gemini-1.5-flash-002", "client:gpt-4-mini", ] export const DEFAULT_EMBEDDINGS_MODEL_CANDIDATES = [ @@ -160,6 +162,7 @@ export const EMOJI_UNDEFINED = "?" export const MODEL_PROVIDER_OPENAI = "openai" export const MODEL_PROVIDER_GITHUB = "github" export const MODEL_PROVIDER_AZURE_OPENAI = "azure" +export const MODEL_PROVIDER_GOOGLE = "google" export const MODEL_PROVIDER_AZURE_SERVERLESS_OPENAI = "azure_serverless" export const MODEL_PROVIDER_AZURE_SERVERLESS_MODELS = "azure_serverless_models" export const MODEL_PROVIDER_OLLAMA = "ollama" @@ -203,6 +206,8 @@ export const DOCS_CONFIGURATION_AICI_URL = "https://microsoft.github.io/genaiscript/reference/scripts/aici/" export const DOCS_CONFIGURATION_ANTHROPIC_URL = "https://microsoft.github.io/genaiscript/getting-started/configuration/#anthropic" +export const DOCS_CONFIGURATION_GOOGLE_URL = + "https://microsoft.github.io/genaiscript/getting-started/configuration/#google" export const DOCS_CONFIGURATION_HUGGINGFACE_URL = "https://microsoft.github.io/genaiscript/getting-started/configuration/#huggingface" export const DOCS_CONFIGURATION_CONTENT_SAFETY_URL = @@ -247,6 +252,11 @@ export const MODEL_PROVIDERS = Object.freeze([ detail: "Anthropic models", url: DOCS_CONFIGURATION_ANTHROPIC_URL, }, + { + id: MODEL_PROVIDER_GOOGLE, + detail: "Google AI", + url: DOCS_CONFIGURATION_GOOGLE_URL, + }, { id: MODEL_PROVIDER_HUGGINGFACE, detail: "Hugging Face models", @@ -357,3 +367,6 @@ export const CHOICE_LOGIT_BIAS = 5 export const SANITIZED_PROMPT_INJECTION = "...prompt injection detected, content removed..." + +export const GOOGLE_API_BASE = + "https://generativelanguage.googleapis.com/v1beta/openai/" diff --git a/packages/core/src/fetch.ts b/packages/core/src/fetch.ts index fa1e3a3e77..1f892db93f 100644 --- a/packages/core/src/fetch.ts +++ b/packages/core/src/fetch.ts @@ -163,12 +163,12 @@ export function traceFetchPost( ? "Bearer ***" // Mask Bearer tokens : "***") // Mask other authorization headers ) - const cmd = `curl ${url} \\ + const cmd = `curl "${url}" \\ +--no-buffer \\ ${Object.entries(headers) .map(([k, v]) => `-H "${k}: ${v}"`) - .join("\\\n")} \\ + .join(" \\\n")} \\ -d '${JSON.stringify(body, null, 2).replace(/'/g, "'\\''")}' ---no-buffer ` if (trace) trace.detailsFenced(`✉️ fetch`, cmd, "bash") else logVerbose(cmd) diff --git a/packages/core/src/genaisrc/system.vision_ask_image.genai.mjs b/packages/core/src/genaisrc/system.vision_ask_image.genai.mjs index cfff81b264..487320dfce 100644 --- a/packages/core/src/genaisrc/system.vision_ask_image.genai.mjs +++ b/packages/core/src/genaisrc/system.vision_ask_image.genai.mjs @@ -40,6 +40,7 @@ defTool( }, { model: "vision", + cache: "vision_ask_image", system: [ "system", "system.assistant", diff --git a/packages/core/src/ollama.ts b/packages/core/src/ollama.ts index f6a44e4df1..1d0eba3c4b 100644 --- a/packages/core/src/ollama.ts +++ b/packages/core/src/ollama.ts @@ -77,7 +77,7 @@ async function listModels( cfg: LanguageModelConfiguration ): Promise { // Create a fetch instance to make HTTP requests - const fetch = await createFetch() + const fetch = await createFetch({ retries: 1 }) // Fetch the list of models from the remote API const res = await fetch(cfg.base.replace("/v1", "/api/tags"), { method: "GET", diff --git a/packages/core/src/openai.ts b/packages/core/src/openai.ts index a8ee9b2b9c..7715cf2333 100644 --- a/packages/core/src/openai.ts +++ b/packages/core/src/openai.ts @@ -203,8 +203,8 @@ export const OpenAIChatCompletion: ChatCompletionHandler = async ( trace.dispatchChange() const fetchHeaders: HeadersInit = { - ...getConfigHeaders(cfg), "Content-Type": "application/json", + ...getConfigHeaders(cfg), ...(headers || {}), } traceFetchPost(trace, url, fetchHeaders as any, postReq) @@ -282,7 +282,9 @@ export const OpenAIChatCompletion: ChatCompletionHandler = async ( numTokens += estimateTokens(delta.content, encoder) chatResp += delta.content tokens.push( - ...serializeChunkChoiceToLogProbs(choice as ChatCompletionChunkChoice) + ...serializeChunkChoiceToLogProbs( + choice as ChatCompletionChunkChoice + ) ) trace.appendToken(delta.content) } else if (Array.isArray(delta.tool_calls)) { @@ -409,7 +411,7 @@ export const OpenAIChatCompletion: ChatCompletionHandler = async ( async function listModels( cfg: LanguageModelConfiguration ): Promise { - const fetch = await createFetch() + const fetch = await createFetch({ retries: 1 }) const res = await fetch(cfg.base + "/models", { method: "GET", headers: { diff --git a/packages/core/src/pricing.json b/packages/core/src/pricing.json index ff62629fc8..6b3cfd61a5 100644 --- a/packages/core/src/pricing.json +++ b/packages/core/src/pricing.json @@ -270,5 +270,50 @@ "azure_serverless_models:ministral-3b": { "price_per_million_input_tokens": 0.04, "price_per_million_output_tokens": 0.04 + }, + "google:gemini-1.5-flash": { + "price_per_million_input_tokens": 0.075, + "price_per_million_output_tokens": 0.3 + }, + "google:gemini-1.5-flash-002": { + "price_per_million_input_tokens": 0.075, + "price_per_million_output_tokens": 0.3 + }, + "google:gemini-1.5-flash-8b": { + "price_per_million_input_tokens": 0.0375, + "price_per_million_output_tokens": 0.15, + "tiers": [ + { + "context_size": 128000, + "price_per_million_input_tokens": 0.075, + "price_per_million_output_tokens": 0.3 + } + ] + }, + "google:gemini-1.5-pro": { + "price_per_million_input_tokens": 1.25, + "price_per_million_output_tokens": 5, + "tiers": [ + { + "context_size": 128000, + "price_per_million_input_tokens": 2.5, + "price_per_million_output_tokens": 10 + } + ] + }, + "google:gemini-1.5-pro-002": { + "price_per_million_input_tokens": 1.25, + "price_per_million_output_tokens": 5, + "tiers": [ + { + "context_size": 128000, + "price_per_million_input_tokens": 2.5, + "price_per_million_output_tokens": 10 + } + ] + }, + "google:gemini-1-pro": { + "price_per_million_input_tokens": 0.5, + "price_per_million_output_tokens": 1.5 } } diff --git a/packages/core/src/tools.ts b/packages/core/src/tools.ts index 18af27b646..cf0be9b861 100644 --- a/packages/core/src/tools.ts +++ b/packages/core/src/tools.ts @@ -2,6 +2,7 @@ import { MODEL_PROVIDER_AZURE_OPENAI, MODEL_PROVIDER_AZURE_SERVERLESS_MODELS, MODEL_PROVIDER_GITHUB, + MODEL_PROVIDER_GOOGLE, MODEL_PROVIDER_OLLAMA, MODEL_PROVIDER_OPENAI, } from "./constants" @@ -38,6 +39,9 @@ export function isToolsSupported(modelId: string): boolean | undefined { [MODEL_PROVIDER_OPENAI]: oai, [MODEL_PROVIDER_AZURE_OPENAI]: oai, [MODEL_PROVIDER_AZURE_SERVERLESS_MODELS]: oai, + [MODEL_PROVIDER_GOOGLE]: { + // all supported + }, [MODEL_PROVIDER_GITHUB]: { "Phi-3.5-mini-instruct": false, }, diff --git a/packages/core/src/types/prompt_template.d.ts b/packages/core/src/types/prompt_template.d.ts index da3e818363..1901c6e903 100644 --- a/packages/core/src/types/prompt_template.d.ts +++ b/packages/core/src/types/prompt_template.d.ts @@ -146,6 +146,12 @@ type ModelType = OptionsOrString< | "anthropic:claude-2.0" | "anthropic:claude-instant-1.2" | "huggingface:microsoft/Phi-3-mini-4k-instruct" + | "google:gemini-1.5-flash" + | "google:gemini-1.5-flash-8b" + | "google:gemini-1.5-flash-002" + | "google:gemini-1.5-pro" + | "google:gemini-1.5-pro-002" + | "google:gemini-1-pro" > type ModelSmallType = OptionsOrString< diff --git a/packages/core/src/usage.ts b/packages/core/src/usage.ts index c3718c2dfd..a8752aa515 100644 --- a/packages/core/src/usage.ts +++ b/packages/core/src/usage.ts @@ -20,6 +20,7 @@ import { MODEL_PROVIDER_AZURE_SERVERLESS_MODELS, MODEL_PROVIDER_AZURE_SERVERLESS_OPENAI, MODEL_PROVIDER_GITHUB, + MODEL_PROVIDER_GOOGLE, MODEL_PROVIDER_OPENAI, } from "./constants" @@ -83,13 +84,15 @@ export function renderCost(value: number) { export function isCosteable(model: string) { const { provider } = parseModelIdentifier(model) - return ( - provider === MODEL_PROVIDER_OPENAI || - provider === MODEL_PROVIDER_AZURE_OPENAI || - provider === MODEL_PROVIDER_AZURE_SERVERLESS_MODELS || - provider === MODEL_PROVIDER_AZURE_SERVERLESS_OPENAI || - provider === MODEL_PROVIDER_ANTHROPIC - ) + const costeableProviders = [ + MODEL_PROVIDER_OPENAI, + MODEL_PROVIDER_AZURE_OPENAI, + MODEL_PROVIDER_AZURE_SERVERLESS_MODELS, + MODEL_PROVIDER_AZURE_SERVERLESS_OPENAI, + MODEL_PROVIDER_ANTHROPIC, + MODEL_PROVIDER_GOOGLE, + ] + return costeableProviders.includes(provider) } /** diff --git a/packages/vscode/src/lmaccess.ts b/packages/vscode/src/lmaccess.ts index 83bd67c922..97e3ad0d6e 100644 --- a/packages/vscode/src/lmaccess.ts +++ b/packages/vscode/src/lmaccess.ts @@ -14,6 +14,7 @@ import { MODEL_PROVIDER_AZURE_SERVERLESS_MODELS, MODEL_PROVIDER_AZURE_SERVERLESS_OPENAI, DOCS_CONFIGURATION_URL, + MODEL_PROVIDER_GOOGLE, } from "../../core/src/constants" import { OpenAIAPIType } from "../../core/src/host" import { parseModelIdentifier } from "../../core/src/models" @@ -29,15 +30,17 @@ async function generateLanguageModelConfiguration( modelId: string ) { const { provider } = parseModelIdentifier(modelId) - if ( - provider === MODEL_PROVIDER_OLLAMA || - provider === MODEL_PROVIDER_LLAMAFILE || - provider === MODEL_PROVIDER_AICI || - provider === MODEL_PROVIDER_AZURE_OPENAI || - provider === MODEL_PROVIDER_AZURE_SERVERLESS_OPENAI || - provider === MODEL_PROVIDER_AZURE_SERVERLESS_MODELS || - provider === MODEL_PROVIDER_LITELLM - ) { + const supportedProviders = [ + MODEL_PROVIDER_OLLAMA, + MODEL_PROVIDER_LLAMAFILE, + MODEL_PROVIDER_AICI, + MODEL_PROVIDER_AZURE_OPENAI, + MODEL_PROVIDER_AZURE_SERVERLESS_OPENAI, + MODEL_PROVIDER_AZURE_SERVERLESS_MODELS, + MODEL_PROVIDER_LITELLM, + MODEL_PROVIDER_GOOGLE, + ] + if (supportedProviders.includes(provider)) { return { provider } }