diff --git a/docs/src/content/docs/getting-started/configuration.mdx b/docs/src/content/docs/getting-started/configuration.mdx index b38e054bfa..5d818799bc 100644 --- a/docs/src/content/docs/getting-started/configuration.mdx +++ b/docs/src/content/docs/getting-started/configuration.mdx @@ -1078,6 +1078,63 @@ script({ }) ``` +## LMStudio + +The `lmstudio` provider connects to the [LMStudio](https://lmstudio.ai/) headless server. +and allows to run local LLMs. + + + +
    + +
  1. + +Install [LMStudio](https://lmstudio.ai/download) (v0.3.5+) + +
  2. + +
  3. + +Open LMStudio, + +
  4. + +
  5. + +Open the [Model Catalog](https://lmstudio.ai/models), +select your model and load it at least once so it is downloaded locally. + +
  6. + +
  7. + +Open the settings (Gearwheel icon) and enable **Enable Local LLM Service**. + +
  8. + +
  9. + +GenAIScript assumes the local server is at `http://localhost:1234/v1` by default. +Add a `LMSTUDIO_API_BASE` environment variable to change the server URL. + +```txt title=".env" +LMSTUDIO_API_BASE=http://localhost:2345/v1 +``` + +
  10. + +
+ +
+ +Find the model **API identifier** in the dialog of loaded models then use that identifier in your script: + +```js '"lmstudio:llama-3.2-1b"' +script({ + model: "lmstudio:llama-3.2-1b-instruct", +}) +``` + ## LocalAI [LocalAI](https://localai.io/) act as a drop-in replacement REST API that’s compatible @@ -1118,9 +1175,9 @@ that allows you to run an LLM locally. The provider is `llamafile` and the model name is ignored. -## Jan, LMStudio, LLaMA.cpp +## Jan, LLaMA.cpp -[Jan](https://jan.ai/), [LMStudio](https://lmstudio.ai/), +[Jan](https://jan.ai/), [LLaMA.cpp](https://github.com/ggerganov/llama.cpp/tree/master/examples/server) also allow running models locally or interfacing with other LLM vendors. diff --git a/packages/core/src/connection.ts b/packages/core/src/connection.ts index bc43bef4d7..1cd7873d61 100644 --- a/packages/core/src/connection.ts +++ b/packages/core/src/connection.ts @@ -30,6 +30,8 @@ import { ALIBABA_BASE, MODEL_PROVIDER_MISTRAL, MISTRAL_API_BASE, + MODEL_PROVIDER_LMSTUDIO, + LMSTUDIO_API_BASE, } from "./constants" import { fileExists, readText, writeText } from "./fs" import { @@ -427,10 +429,14 @@ export async function parseTokenFromEnv( } if (provider === MODEL_PROVIDER_LLAMAFILE) { + const base = + findEnvVar(env, "LLAMAFILE", BASE_SUFFIX)?.value || + LLAMAFILE_API_BASE + if (!URL.canParse(base)) throw new Error(`${base} must be a valid URL`) return { provider, model, - base: LLAMAFILE_API_BASE, + base, token: "llamafile", type: "openai", source: "default", @@ -438,16 +444,33 @@ export async function parseTokenFromEnv( } if (provider === MODEL_PROVIDER_LITELLM) { + const base = + findEnvVar(env, "LITELLM", BASE_SUFFIX)?.value || LITELLM_API_BASE + if (!URL.canParse(base)) throw new Error(`${base} must be a valid URL`) return { provider, model, - base: LITELLM_API_BASE, + base, token: "litellm", type: "openai", source: "default", } } + if (provider === MODEL_PROVIDER_LMSTUDIO) { + const base = + findEnvVar(env, "LMSTUDIO", BASE_SUFFIX)?.value || LMSTUDIO_API_BASE + if (!URL.canParse(base)) throw new Error(`${base} must be a valid URL`) + return { + provider, + model, + base, + token: "lmstudio", + type: "openai", + source: "env: LMSTUDIO_API_...", + } + } + if (provider === MODEL_PROVIDER_TRANSFORMERS) { return { provider, diff --git a/packages/core/src/constants.ts b/packages/core/src/constants.ts index 3a9c4ef6da..0c367aeb53 100644 --- a/packages/core/src/constants.ts +++ b/packages/core/src/constants.ts @@ -155,6 +155,7 @@ export const GOOGLE_API_BASE = export const ALIBABA_BASE = "https://dashscope-intl.aliyuncs.com/compatible-mode/v1" export const MISTRAL_API_BASE = "https://api.mistral.ai/v1" +export const LMSTUDIO_API_BASE = "http://localhost:1234/v1" export const PROMPTFOO_CACHE_PATH = ".genaiscript/cache/tests" export const PROMPTFOO_CONFIG_DIR = ".genaiscript/config/tests" @@ -184,6 +185,7 @@ export const MODEL_PROVIDER_HUGGINGFACE = "huggingface" export const MODEL_PROVIDER_TRANSFORMERS = "transformers" export const MODEL_PROVIDER_ALIBABA = "alibaba" export const MODEL_PROVIDER_MISTRAL = "mistral" +export const MODEL_PROVIDER_LMSTUDIO = "lmstudio" export const TRACE_FILE_PREVIEW_MAX_LENGTH = 240 @@ -208,6 +210,8 @@ export const DOCS_CONFIGURATION_AZURE_MODELS_SERVERLESS_URL = "https://microsoft.github.io/genaiscript/getting-started/configuration/#azure_serverless_models" export const DOCS_CONFIGURATION_OLLAMA_URL = "https://microsoft.github.io/genaiscript/getting-started/configuration/#ollama" +export const DOCS_CONFIGURATION_LMSTUDIO_URL = + "https://microsoft.github.io/genaiscript/getting-started/configuration/#lmstudio" export const DOCS_CONFIGURATION_LLAMAFILE_URL = "https://microsoft.github.io/genaiscript/getting-started/configuration/#llamafile" export const DOCS_CONFIGURATION_LITELLM_URL = @@ -295,6 +299,11 @@ export const MODEL_PROVIDERS = Object.freeze([ detail: "Ollama local model", url: DOCS_CONFIGURATION_OLLAMA_URL, }, + { + id: MODEL_PROVIDER_LMSTUDIO, + detail: "LM Studio local server", + url: DOCS_CONFIGURATION_LMSTUDIO_URL, + }, { id: MODEL_PROVIDER_ALIBABA, detail: "Alibaba models", diff --git a/packages/vscode/src/lmaccess.ts b/packages/vscode/src/lmaccess.ts index 46dcd5ed3c..bea3b4af50 100644 --- a/packages/vscode/src/lmaccess.ts +++ b/packages/vscode/src/lmaccess.ts @@ -16,6 +16,7 @@ import { DOCS_CONFIGURATION_URL, MODEL_PROVIDER_GOOGLE, MODEL_PROVIDER_ALIBABA, + MODEL_PROVIDER_LMSTUDIO, } from "../../core/src/constants" import { OpenAIAPIType } from "../../core/src/host" import { parseModelIdentifier } from "../../core/src/models" @@ -39,6 +40,7 @@ async function generateLanguageModelConfiguration( MODEL_PROVIDER_AZURE_SERVERLESS_OPENAI, MODEL_PROVIDER_AZURE_SERVERLESS_MODELS, MODEL_PROVIDER_LITELLM, + MODEL_PROVIDER_LMSTUDIO, MODEL_PROVIDER_GOOGLE, MODEL_PROVIDER_ALIBABA, ]