From 11a12e70e61051095b0620b2d118cef509d4e34d Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Sat, 21 Dec 2024 08:16:45 +0100 Subject: [PATCH] refactor llm info (#961) --- .github/workflows/anthropic.yml | 6 +- .github/workflows/google.yml | 2 +- .github/workflows/huggingface.yml | 37 + .github/workflows/openai.yml | 22 +- THIRD_PARTY_LICENSES.md | 82 +- docs/package.json | 4 +- docs/public/schemas/llms.json | 51 + docs/src/components/LLMProviderFeatures.astro | 55 +- .../docs/getting-started/configuration.mdx | 9 +- .../content/docs/reference/cli/commands.md | 39 +- docs/src/content/docs/reference/cli/run.mdx | 20 +- .../docs/reference/scripts/model-aliases.mdx | 7 + docs/yarn.lock | 929 ++++++--- package.json | 4 +- packages/cli/package.json | 6 +- packages/cli/src/cli.ts | 53 +- packages/cli/src/convert.ts | 2 +- packages/cli/src/info.ts | 9 +- packages/cli/src/modealias.ts | 16 - packages/cli/src/modelalias.ts | 38 + packages/cli/src/nodehost.ts | 52 +- packages/cli/src/parse.ts | 3 +- packages/cli/src/run.ts | 2 +- packages/cli/src/test.ts | 10 +- packages/core/package.json | 8 +- packages/core/src/chat.ts | 36 +- packages/core/src/constants.ts | 51 +- packages/core/src/llms.json | 153 +- packages/core/src/llms.ts | 35 + packages/core/src/promptrunner.ts | 2 +- packages/core/src/server/messages.ts | 1 + packages/core/src/testhost.ts | 14 +- packages/core/src/types/prompt_template.d.ts | 1 + packages/core/src/util.ts | 14 + packages/core/src/vectorsearch.ts | 1 - packages/sample/genaisrc/judge.genai.mts | 80 + packages/sample/package.json | 2 +- packages/sample/src/cli.test.ts | 9 +- slides/package.json | 2 +- slides/yarn.lock | 1747 +++++++++-------- yarn.lock | 839 ++++---- 41 files changed, 2721 insertions(+), 1732 deletions(-) create mode 100644 .github/workflows/huggingface.yml delete mode 100644 packages/cli/src/modealias.ts create mode 100644 packages/cli/src/modelalias.ts create mode 100644 packages/core/src/llms.ts create mode 100644 packages/sample/genaisrc/judge.genai.mts diff --git a/.github/workflows/anthropic.yml b/.github/workflows/anthropic.yml index ccd972dfa7..5ec1442d0f 100644 --- a/.github/workflows/anthropic.yml +++ b/.github/workflows/anthropic.yml @@ -32,10 +32,6 @@ jobs: - name: compile run: yarn compile - name: poem - run: yarn run:script poem --model anthropic:claude-3-5-sonnet-20240620 --out-trace $GITHUB_STEP_SUMMARY - env: - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - - name: cache - run: yarn run:script summarize-cached --model anthropic:claude-3-5-sonnet-20240620 --small-model anthropic:claude-3-5-sonnet-20240620 + run: yarn run:script poem -p anthropic --out-trace $GITHUB_STEP_SUMMARY env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} diff --git a/.github/workflows/google.yml b/.github/workflows/google.yml index 6f61fd4a7a..a3bdd845d2 100644 --- a/.github/workflows/google.yml +++ b/.github/workflows/google.yml @@ -32,6 +32,6 @@ jobs: - name: compile run: yarn compile - name: poem - run: yarn run:script poem --model google:gemini-1.5-flash --out-trace $GITHUB_STEP_SUMMARY + run: yarn run:script poem -p google --out-trace $GITHUB_STEP_SUMMARY env: GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} diff --git a/.github/workflows/huggingface.yml b/.github/workflows/huggingface.yml new file mode 100644 index 0000000000..85f5c3f92b --- /dev/null +++ b/.github/workflows/huggingface.yml @@ -0,0 +1,37 @@ +name: huggingface tests +on: + workflow_dispatch: + release: + types: + - published + pull_request: + paths: + - yarn.lock + - ".github/workflows/huggingface.yml" + - "packages/core/**/*" + - "packages/cli/**/*" + - "packages/samples/**/*" +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-huggingface + cancel-in-progress: true +jobs: + tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: "recursive" + fetch-depth: 0 + - uses: actions/setup-node@v4 + with: + node-version: "20" + cache: yarn + - run: yarn install --frozen-lockfile + - name: typecheck + run: yarn typecheck + - name: compile + run: yarn compile + - name: poem + run: yarn run:script poem -p huggingface --out-trace $GITHUB_STEP_SUMMARY + env: + HUGGINGFACE_API_KEY: ${{ secrets.HUGGINGFACE_API_KEY }} diff --git a/.github/workflows/openai.yml b/.github/workflows/openai.yml index 92b58af063..3e55fe42f1 100644 --- a/.github/workflows/openai.yml +++ b/.github/workflows/openai.yml @@ -1,24 +1,8 @@ name: openai smoke tests on: workflow_dispatch: - # pull_request: - # paths: - # - yarn.lock - # - ".github/workflows/github-models.yml" - # - "packages/core/**/*" - # - "packages/cli/**/*" - # - "packages/samples/**/*" - # push: - # branches: - # - main - # paths: - # - yarn.lock - # - ".github/workflows/github-models.yml" - # - "packages/core/**/*" - # - "packages/cli/**/*" - # - "packages/samples/**/*" concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-github-models + group: ${{ github.workflow }}-${{ github.ref }}-openai cancel-in-progress: true jobs: tests: @@ -37,8 +21,8 @@ jobs: run: yarn typecheck - name: compile run: yarn compile - - name: run summarize openai:gpt-3.5-turbo - run: yarn test:summarize --model openai:gpt-3.5-turbo --out ./temp + - name: run summarize + run: yarn test:summarize -p openai --out-trace $GITHUB_STEP_SUMMARY env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} OPENAI_API_TYPE: ${{ secrets.OPENAI_API_TYPE }} diff --git a/THIRD_PARTY_LICENSES.md b/THIRD_PARTY_LICENSES.md index dfe3d98cee..2cb314c4ce 100644 --- a/THIRD_PARTY_LICENSES.md +++ b/THIRD_PARTY_LICENSES.md @@ -190,7 +190,7 @@ The following npm packages may be included in this product: - @aws-crypto/sha256-browser@5.2.0 - @aws-crypto/supports-web-crypto@5.2.0 - - @huggingface/transformers@3.2.0 + - @huggingface/transformers@3.2.1 - flatbuffers@1.12.0 - long@5.2.3 @@ -1954,16 +1954,16 @@ Apache License The following npm packages may be included in this product: - - @aws-sdk/client-cognito-identity@3.714.0 - - @aws-sdk/client-sso-oidc@3.714.0 - - @aws-sdk/client-sso@3.714.0 - - @aws-sdk/client-sts@3.714.0 - - @aws-sdk/credential-provider-cognito-identity@3.714.0 + - @aws-sdk/client-cognito-identity@3.716.0 + - @aws-sdk/client-sso-oidc@3.716.0 + - @aws-sdk/client-sso@3.716.0 + - @aws-sdk/client-sts@3.716.0 + - @aws-sdk/credential-provider-cognito-identity@3.716.0 - @aws-sdk/util-user-agent-browser@3.714.0 - - @aws-sdk/util-user-agent-node@3.714.0 + - @aws-sdk/util-user-agent-node@3.716.0 - @smithy/eventstream-codec@2.2.0 - @smithy/eventstream-codec@3.1.10 - - @smithy/middleware-retry@3.0.30 + - @smithy/middleware-retry@3.0.31 - @smithy/querystring-builder@2.2.0 - @smithy/querystring-builder@3.0.11 - @smithy/querystring-parser@2.2.0 @@ -1971,8 +1971,8 @@ The following npm packages may be included in this product: - @smithy/service-error-classification@3.0.11 - @smithy/url-parser@2.2.0 - @smithy/url-parser@3.0.11 - - @smithy/util-defaults-mode-browser@3.0.30 - - @smithy/util-defaults-mode-node@3.0.30 + - @smithy/util-defaults-mode-browser@3.0.31 + - @smithy/util-defaults-mode-node@3.0.31 These packages each contain the following license: @@ -2182,7 +2182,7 @@ Apache License The following npm package may be included in this product: - - @aws-sdk/client-bedrock-runtime@3.714.0 + - @aws-sdk/client-bedrock-runtime@3.716.0 This package contains the following license: @@ -2394,7 +2394,7 @@ The following npm packages may be included in this product: - @aws-sdk/middleware-host-header@3.714.0 - @aws-sdk/middleware-recursion-detection@3.714.0 - - @aws-sdk/middleware-user-agent@3.714.0 + - @aws-sdk/middleware-user-agent@3.716.0 - @smithy/core@2.5.5 - @smithy/eventstream-serde-browser@3.0.14 - @smithy/eventstream-serde-config-resolver@3.0.11 @@ -2408,7 +2408,7 @@ The following npm packages may be included in this product: - @smithy/protocol-http@3.3.0 - @smithy/protocol-http@4.1.8 - @smithy/smithy-client@2.5.1 - - @smithy/smithy-client@3.5.0 + - @smithy/smithy-client@3.5.1 - @smithy/types@2.12.0 - @smithy/types@3.7.2 @@ -2861,7 +2861,7 @@ MIT License The following npm package may be included in this product: - - genaiscript-vscode@1.86.0 + - genaiscript-vscode@1.86.4 This package contains the following license: @@ -4161,10 +4161,10 @@ Apache License The following npm packages may be included in this product: - - @aws-sdk/credential-provider-env@3.714.0 - - @aws-sdk/credential-provider-ini@3.714.0 - - @aws-sdk/credential-provider-node@3.714.0 - - @aws-sdk/credential-providers@3.715.0 + - @aws-sdk/credential-provider-env@3.716.0 + - @aws-sdk/credential-provider-ini@3.716.0 + - @aws-sdk/credential-provider-node@3.716.0 + - @aws-sdk/credential-providers@3.716.0 - @aws-sdk/region-config-resolver@3.714.0 - @aws-sdk/token-providers@3.714.0 - @aws-sdk/types@3.714.0 @@ -4182,7 +4182,7 @@ The following npm packages may be included in this product: - @smithy/is-array-buffer@3.0.0 - @smithy/middleware-content-length@3.0.13 - @smithy/middleware-endpoint@2.5.1 - - @smithy/middleware-endpoint@3.2.5 + - @smithy/middleware-endpoint@3.2.6 - @smithy/middleware-stack@2.2.0 - @smithy/middleware-stack@3.0.11 - @smithy/node-http-handler@2.5.0 @@ -4416,9 +4416,9 @@ Apache License The following npm packages may be included in this product: - - @aws-sdk/credential-provider-process@3.714.0 - - @aws-sdk/credential-provider-sso@3.714.0 - - @aws-sdk/credential-provider-web-identity@3.714.0 + - @aws-sdk/credential-provider-process@3.716.0 + - @aws-sdk/credential-provider-sso@3.716.0 + - @aws-sdk/credential-provider-web-identity@3.716.0 These packages each contain the following license: @@ -5318,8 +5318,8 @@ END OF TERMS AND CONDITIONS The following npm packages may be included in this product: - - @aws-sdk/core@3.714.0 - - @aws-sdk/credential-provider-http@3.714.0 + - @aws-sdk/core@3.716.0 + - @aws-sdk/credential-provider-http@3.716.0 These packages each contain the following license: @@ -6390,20 +6390,20 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. The following npm packages may be included in this product: - - @inquirer/checkbox@4.0.3 - - @inquirer/confirm@5.1.0 - - @inquirer/core@10.1.1 - - @inquirer/editor@4.2.0 - - @inquirer/expand@4.0.3 - - @inquirer/figures@1.0.8 - - @inquirer/input@4.1.0 - - @inquirer/number@3.0.3 - - @inquirer/password@4.0.3 - - @inquirer/prompts@7.2.0 - - @inquirer/rawlist@4.0.3 - - @inquirer/search@3.0.3 - - @inquirer/select@4.0.3 - - @inquirer/type@3.0.1 + - @inquirer/checkbox@4.0.4 + - @inquirer/confirm@5.1.1 + - @inquirer/core@10.1.2 + - @inquirer/editor@4.2.1 + - @inquirer/expand@4.0.4 + - @inquirer/figures@1.0.9 + - @inquirer/input@4.1.1 + - @inquirer/number@3.0.4 + - @inquirer/password@4.0.4 + - @inquirer/prompts@7.2.1 + - @inquirer/rawlist@4.0.4 + - @inquirer/search@3.0.4 + - @inquirer/select@4.0.4 + - @inquirer/type@3.0.2 These packages each contain the following license: @@ -7891,9 +7891,9 @@ The following npm packages may be included in this product: - @tokenizer/token@0.3.0 - agent-base@6.0.2 - eastasianwidth@0.2.0 - - genaiscript-core-internal@1.86.0 - - genaiscript-sample@1.86.0 - - genaiscript@1.86.0 + - genaiscript-core-internal@1.86.4 + - genaiscript-sample@1.86.4 + - genaiscript@1.86.4 - https-proxy-agent@5.0.1 - isarray@1.0.0 - javascript-natural-sort@0.7.1 diff --git a/docs/package.json b/docs/package.json index c759f09fbb..31a4fe1670 100644 --- a/docs/package.json +++ b/docs/package.json @@ -21,8 +21,8 @@ "dependencies": {}, "devDependencies": { "@astrojs/check": "^0.9.4", - "@astrojs/starlight": "^0.30.2", - "astro": "^5.0.9", + "@astrojs/starlight": "^0.30.3", + "astro": "^5.1.1", "rehype-mermaid": "^3.0.0", "starlight-blog": "^0.16.0", "starlight-links-validator": "^0.14.0", diff --git a/docs/public/schemas/llms.json b/docs/public/schemas/llms.json index b750191fab..bc004791db 100644 --- a/docs/public/schemas/llms.json +++ b/docs/public/schemas/llms.json @@ -58,6 +58,48 @@ "bearerToken": { "type": "boolean", "description": "Indicates if bearer token is supported" + }, + "aliases": { + "type": "object", + "description": "List of model aliases for the provider", + "properties": { + "large": { + "type": "string", + "description": "Alias for large model" + }, + "small": { + "type": "string", + "description": "Alias for small model" + }, + "vision": { + "type": "string", + "description": "Alias for vision model" + }, + "reasoning": { + "type": "string", + "description": "Alias for reasoning model" + }, + "reasoning_small": { + "type": "string", + "description": "Alias for reasoning small model" + }, + "long": { + "type": "string", + "description": "Alias for long model" + }, + "agent": { + "type": "string", + "description": "Alias for agent model" + }, + "memory": { + "type": "string", + "description": "Alias for memory model" + }, + "embeddings": { + "type": "string", + "description": "Alias for embeddings model" + } + } } }, "additionalProperties": false, @@ -65,6 +107,15 @@ } } }, + "aliases": { + "type": "object", + "additionalProperties": true, + "patternProperties": { + "^[a-zA-Z0-9:_-]+$": { + "type": "string" + } + } + }, "pricings": { "type": "object", "additionalProperties": false, diff --git a/docs/src/components/LLMProviderFeatures.astro b/docs/src/components/LLMProviderFeatures.astro index 164a625a9b..104ff98f0a 100644 --- a/docs/src/components/LLMProviderFeatures.astro +++ b/docs/src/components/LLMProviderFeatures.astro @@ -5,8 +5,11 @@ interface Props { provider: string } const { provider } = Astro.props -const info: Record & { openaiCompatibility?: string, limitations?: boolean } = - LLMS.providers.find(({ id }) => id === provider) as any +const info: Record & { + openaiCompatibility?: string + limitations?: boolean + aliases?: Record +} = LLMS.providers.find(({ id }) => id === provider) as any if (!info) { throw new Error(`Provider ${provider} not found`) } @@ -17,6 +20,9 @@ const features: Record = { topP: { name: "top_p ignored", }, + logitBias: { + name: "logit_bias ignored", + }, logprobs: { name: "logprobs (and top logprobs) ignored", }, @@ -26,31 +32,58 @@ const features: Record = { tools: { name: "Tools implemented as fallback tools automatically.", }, + prediction: { + name: "Ignore prediction of output tokens", + }, } -const { openaiCompatibility, limitations, ...rest } = info +const { openaiCompatibility, limitations, aliases, bearerToken, ...rest } = info const unsupported = Object.keys(rest) .sort() .map((id) => ({ id, supported: info[id] })) .filter(({ supported }) => supported === false) --- +{ + aliases && ( + <> +

Aliases

+

+ The following model aliases are attempted by default in + GenAIScript. +

+ + + + + + {Object.entries(aliases).map(([key, value]) => ( + + + + + ))} +
AliasModel identifier
{key}{value}
+ + ) +} + { openaiCompatibility || limitations || unsupported?.length > 0 ? ( <> -

Limitations

+

Limitations

    {!!limitations &&
  • {limitations}
    • } {!!openaiCompatibility && (
  • - Uses OpenAI compatibility layer + Uses{" "} + + OpenAI compatibility layer +
    • )} - {Object.keys(features) - .map((id) => ({ id, supported: info[id] })) - .filter(({ supported }) => supported === false) - .map(({ id }) => ( -
  • {features[id]?.name || id}
    • - ))} + {unsupported.map(({ id }) => ( +
  • {features[id]?.name || id}
    • + ))}
) : null diff --git a/docs/src/content/docs/getting-started/configuration.mdx b/docs/src/content/docs/getting-started/configuration.mdx index 1123382b17..5c6155956f 100644 --- a/docs/src/content/docs/getting-started/configuration.mdx +++ b/docs/src/content/docs/getting-started/configuration.mdx @@ -64,6 +64,13 @@ GENAISCRIPT_MODEL_SMALL="azure_serverless:..." GENAISCRIPT_MODEL_VISION="azure_serverless:..." ``` +You can also configure the default aliases for a given LLM provider by using the `--provider` argument in the CLI. +The default are documented in this page and printed to the console output. + +```sh +genaiscript run ... --provider anthropic +``` + ### Model aliases In fact, you can define any alias for your model (only alphanumeric characters are allowed) @@ -984,7 +991,7 @@ script({ - + ## Alibaba Cloud diff --git a/docs/src/content/docs/reference/cli/commands.md b/docs/src/content/docs/reference/cli/commands.md index 120ff8111d..1099cc98f3 100644 --- a/docs/src/content/docs/reference/cli/commands.md +++ b/docs/src/content/docs/reference/cli/commands.md @@ -16,6 +16,7 @@ Usage: genaiscript run [options]