refactor llm info

docs/public/schemas/llms.json

@@ -58,13 +58,64 @@
                     "bearerToken": {
                         "type": "boolean",
                         "description": "Indicates if bearer token is supported"
+                    },
+                    "aliases": {
+                        "type": "object",
+                        "description": "List of model aliases for the provider",
+                        "properties": {
+                            "large": {
+                                "type": "string",
+                                "description": "Alias for large model"
+                            },
+                            "small": {
+                                "type": "string",
+                                "description": "Alias for small model"
+                            },
+                            "vision": {
+                                "type": "string",
+                                "description": "Alias for vision model"
+                            },
+                            "reasoning": {
+                                "type": "string",
+                                "description": "Alias for reasoning model"
+                            },
+                            "reasoning_small": {
+                                "type": "string",
+                                "description": "Alias for reasoning small model"
+                            },
+                            "long": {
+                                "type": "string",
+                                "description": "Alias for long model"
+                            },
+                            "agent": {
+                                "type": "string",
+                                "description": "Alias for agent model"
+                            },
+                            "memory": {
+                                "type": "string",
+                                "description": "Alias for memory model"
+                            },
+                            "embeddings": {
+                                "type": "string",
+                                "description": "Alias for embeddings model"
+                            }
+                        }
                     }
                 },
                 "additionalProperties": false,
                 "required": ["id", "detail"]
             }
         }
     },
+    "aliases": {
+        "type": "object",
+        "additionalProperties": true,
+        "patternProperties": {
+            "^[a-zA-Z0-9:_-]+$": {
+                "type": "string"
+            }
+        }
+    },
     "pricings": {
         "type": "object",
         "additionalProperties": false,

docs/src/content/docs/reference/cli/commands.md

@@ -249,9 +249,20 @@
 
 Options:
   -t, --token  show token
   -h, --help   display help for command
 ```
 
+## `models`
+
+```
+Usage: genaiscript models [options]
+
+List model information
+
+Options:
+  -h, --help  display help for command
+```
+
 ## `cache`
 
 ```

packages/cli/src/cli.ts

@@ -24,7 +24,7 @@ import {
 } from "./parse" // Parsing functions
 import { compileScript, createScript, fixScripts, listScripts } from "./scripts" // Script utilities
 import { codeQuery } from "./codequery" // Code parsing and query execution
-import { envInfo, modelInfo, systemInfo } from "./info" // Information utilities
+import { envInfo, modelsInfo, scriptModelInfo, systemInfo } from "./info" // Information utilities
 import { scriptTestList, scriptTestsView, scriptsTest } from "./test" // Test functions
 import { cacheClear } from "./cache" // Cache management
 import "node:console" // Importing console for side effects
@@ -320,7 +320,11 @@ export async function cli() {
         .description("List model connection information for scripts")
         .argument("[script]", "Script id or file")
         .option("-t, --token", "show token")
-        .action(modelInfo) // Action to show model information
+        .action(scriptModelInfo) // Action to show model information
+
+    const models = program.command("models")
+        .description("List model information")
+        .action(modelsInfo)
 
     // Define 'cache' command for cache management
     const cache = program.command("cache").description("Cache management")
@@ -446,5 +450,6 @@ export async function cli() {
         .option("-e, --error", "show errors")
         .option("-m, --models", "show models if possible")
         .action(envInfo) // Action to show environment information
+
     program.parse() // Parse command-line arguments
 }

packages/cli/src/info.ts

@@ -117,7 +117,10 @@ async function resolveScriptsConnectionInfo(
  * @param script - The specific script ID or filename to filter by (optional).
  * @param options - Configuration options, including whether to show tokens.
  */
-export async function modelInfo(script: string, options?: { token?: boolean }) {
+export async function scriptModelInfo(
+    script: string,
+    options?: { token?: boolean }
+) {
     const prj = await buildProject()
     const templates = prj.scripts.filter(
         (t) =>
@@ -128,3 +131,7 @@ export async function modelInfo(script: string, options?: { token?: boolean }) {
     const info = await resolveScriptsConnectionInfo(templates, options)
     console.log(YAMLStringify(info))
 }
+
+export async function modelsInfo() {
+    console.log(YAML.stringify(runtimeHost.modelAliases))
+}

packages/cli/src/nodehost.ts

@@ -17,27 +17,16 @@ import {
     parseTokenFromEnv,
 } from "../../core/src/connection"
 import {
-    DEFAULT_LARGE_MODEL,
     MODEL_PROVIDER_AZURE_OPENAI,
     SHELL_EXEC_TIMEOUT,
-    DEFAULT_EMBEDDINGS_MODEL,
-    DEFAULT_SMALL_MODEL,
     AZURE_COGNITIVE_SERVICES_TOKEN_SCOPES,
     MODEL_PROVIDER_AZURE_SERVERLESS_MODELS,
     AZURE_AI_INFERENCE_TOKEN_SCOPES,
     MODEL_PROVIDER_AZURE_SERVERLESS_OPENAI,
     DOT_ENV_FILENAME,
-    DEFAULT_VISION_MODEL,
     LARGE_MODEL_ID,
     SMALL_MODEL_ID,
-    DEFAULT_SMALL_MODEL_CANDIDATES,
-    DEFAULT_LARGE_MODEL_CANDIDATES,
-    DEFAULT_EMBEDDINGS_MODEL_CANDIDATES,
-    DEFAULT_VISION_MODEL_CANDIDATES,
-    DEFAULT_REASONING_MODEL,
-    DEFAULT_REASONING_SMALL_MODEL,
-    DEFAULT_REASONING_SMALL_MODEL_CANDIDATES,
-    DEFAULT_REASONING_MODEL_CANDIDATES,
+    VISION_MODEL_ID,
 } from "../../core/src/constants"
 import { tryReadText } from "../../core/src/fs"
 import {
@@ -54,7 +43,7 @@ import {
     ModelConfiguration,
 } from "../../core/src/host"
 import { TraceOptions } from "../../core/src/trace"
-import { logError, logVerbose } from "../../core/src/util"
+import { deleteEmptyValues, logError, logVerbose } from "../../core/src/util"
 import { parseModelIdentifier } from "../../core/src/models"
 import { LanguageModel } from "../../core/src/chat"
 import { errorMessage, NotSupportedError } from "../../core/src/error"
@@ -73,6 +62,7 @@ import { resolveGlobalConfiguration } from "../../core/src/config"
 import { HostConfiguration } from "../../core/src/hostconfiguration"
 import { resolveLanguageModel } from "../../core/src/lm"
 import { CancellationOptions } from "../../core/src/cancellation"
+import { defaultModelConfigurations } from "../../core/src/llms"
 
 class NodeServerManager implements ServerManager {
     async start(): Promise<void> {
@@ -97,41 +87,7 @@ export class NodeHost implements RuntimeHost {
         "default" | "cli" | "env" | "config",
         Omit<ModelConfigurations, "large" | "small" | "vision" | "embeddings">
     > = {
-        default: {
-            large: {
-                model: DEFAULT_LARGE_MODEL,
-                source: "default",
-                candidates: DEFAULT_LARGE_MODEL_CANDIDATES,
-            },
-            small: {
-                model: DEFAULT_SMALL_MODEL,
-                source: "default",
-                candidates: DEFAULT_SMALL_MODEL_CANDIDATES,
-            },
-            vision: {
-                model: DEFAULT_VISION_MODEL,
-                source: "default",
-                candidates: DEFAULT_VISION_MODEL_CANDIDATES,
-            },
-            embeddings: {
-                model: DEFAULT_EMBEDDINGS_MODEL,
-                source: "default",
-                candidates: DEFAULT_EMBEDDINGS_MODEL_CANDIDATES,
-            },
-            reasoning: {
-                model: DEFAULT_REASONING_MODEL,
-                source: "default",
-                candidates: DEFAULT_REASONING_MODEL_CANDIDATES,
-            },
-            ["reasoning_small"]: {
-                model: DEFAULT_REASONING_SMALL_MODEL,
-                source: "default",
-                candidates: DEFAULT_REASONING_SMALL_MODEL_CANDIDATES,
-            },
-            long: { model: LARGE_MODEL_ID, source: "default" },
-            agent: { model: LARGE_MODEL_ID, source: "default" },
-            memory: { model: SMALL_MODEL_ID, source: "default" },
-        },
+        default: defaultModelConfigurations(),
         cli: {},
         env: {},
         config: {},

packages/cli/src/parse.ts

@@ -16,7 +16,6 @@ import { YAMLParse, YAMLStringify } from "../../core/src/yaml"
 import { resolveTokenEncoder } from "../../core/src/encoders"
 import {
     CSV_REGEX,
-    DEFAULT_LARGE_MODEL,
     INI_REGEX,
     JSON5_REGEX,
     MD_REGEX,
@@ -203,7 +202,7 @@ export async function parseTokens(
     filesGlobs: string[],
     options: { excludedFiles: string[]; model: string }
 ) {
-    const { model = DEFAULT_LARGE_MODEL } = options || {}
+    const { model } = options || {}
     const { encode: encoder } = await resolveTokenEncoder(model)
 
     const files = await expandFiles(filesGlobs, options?.excludedFiles)

packages/core/src/chat.ts

@@ -785,7 +785,7 @@ async function choicesToLogitBias(
             disableFallback: true,
         })) || {}
     if (!encode) {
-        logWarn(
+        logVerbose(
             `unabled to compute logit bias, no token encoder found for ${model}`
         )
         trace.warn(
@@ -1052,18 +1052,20 @@ export function tracePromptResult(
     const { text } = resp || {}
 
     // try to sniff the output type
-    const language = JSON5TryParse(text)
-        ? "json"
-        : XMLTryParse(text)
-          ? "xml"
-          : /^(-|\*|#+|```)\s/im.test(text)
-            ? "markdown"
-            : "text"
-    trace.detailsFenced(`🔠 output`, text, language)
-    if (language === "markdown")
-        trace.appendContent(
-            "\n\n" + HTMLEscape(prettifyMarkdown(text)) + "\n\n"
-        )
+    if (text !== undefined) {
+        const language = JSON5TryParse(text)
+            ? "json"
+            : XMLTryParse(text)
+              ? "xml"
+              : /^(-|\*|#+|```)\s/im.test(text)
+                ? "markdown"
+                : "text"
+        trace.detailsFenced(`🔠 output`, text, language)
+        if (language === "markdown")
+            trace.appendContent(
+                "\n\n" + HTMLEscape(prettifyMarkdown(text)) + "\n\n"
+            )
+    }
 }
 
 export function appendUserMessage(

packages/core/src/constants.ts

@@ -57,56 +57,6 @@ export const SMALL_MODEL_ID = "small"
 export const LARGE_MODEL_ID = "large"
 export const VISION_MODEL_ID = "vision"
 export const DEFAULT_FENCE_FORMAT: FenceFormat = "xml"
-export const DEFAULT_LARGE_MODEL = "openai:gpt-4o"
-export const DEFAULT_LARGE_MODEL_CANDIDATES = [
-    "azure_serverless:gpt-4o",
-    DEFAULT_LARGE_MODEL,
-    "google:gemini-1.5-pro-latest",
-    "anthropic:claude-2.1",
-    "mistral:mistral-large-latest",
-    "github:gpt-4o",
-    "client:gpt-4",
-]
-export const DEFAULT_VISION_MODEL = "openai:gpt-4o"
-export const DEFAULT_VISION_MODEL_CANDIDATES = [
-    "azure_serverless:gpt-4o",
-    DEFAULT_VISION_MODEL,
-    "google:gemini-1.5-flash-latest",
-    "anthropic:claude-2.1",
-    "github:gpt-4o",
-]
-export const DEFAULT_SMALL_MODEL = "openai:gpt-4o-mini"
-export const DEFAULT_SMALL_MODEL_CANDIDATES = [
-    "azure_serverless:gpt-4o-mini",
-    DEFAULT_SMALL_MODEL,
-    "google:gemini-1.5-flash-latest",
-    "anthropic:claude-instant-1.2",
-    "mistral:mistral-small-latest",
-    "github:gpt-4o-mini",
-    "client:gpt-4-mini",
-]
-export const DEFAULT_EMBEDDINGS_MODEL_CANDIDATES = [
-    "azure:text-embedding-3-small",
-    "azure:text-embedding-2-small",
-    "openai:text-embedding-3-small",
-    "github:text-embedding-3-small",
-    "client:text-embedding-3-small",
-]
-export const DEFAULT_REASONING_SMALL_MODEL = "openai:o1-mini"
-export const DEFAULT_REASONING_SMALL_MODEL_CANDIDATES = [
-    "azure_serverless:o1-mini",
-    DEFAULT_REASONING_SMALL_MODEL,
-    "github:o1-mini",
-    "client:o1-mini",
-]
-export const DEFAULT_REASONING_MODEL = "openai:o1"
-export const DEFAULT_REASONING_MODEL_CANDIDATES = [
-    "azure_serverless:o1-preview",
-    DEFAULT_REASONING_MODEL,
-    "github:o1-preview",
-    "client:o1-preview",
-]
-export const DEFAULT_EMBEDDINGS_MODEL = "openai:text-embedding-ada-002"
 export const DEFAULT_TEMPERATURE = 0.8
 export const BUILTIN_PREFIX = "_builtin/"
 export const CACHE_LLMREQUEST_PREFIX = "genaiscript/cache/llm/"