fix: 🐛 add fallback for token encoder & assertions

microsoft · Dec 3, 2024 · e89d84e · e89d84e
1 parent e861545
commit e89d84e
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 1 deletion.
diff --git a/packages/core/src/chat.ts b/packages/core/src/chat.ts
@@ -749,7 +749,7 @@ async function choicesToLogitBias(
     if (!choices?.length) return undefined
     const { encode } = await resolveTokenEncoder(model, {
         disableFallback: true,
-    })
+    }) || {}
     if (!encode) {
         trace.error(
             `unabled to compute logit bias, no token encoder found for ${model}`

diff --git a/packages/core/src/encoders.ts b/packages/core/src/encoders.ts
@@ -5,6 +5,7 @@ import path from "node:path"
 import { addLineNumbers, indexToLineNumber } from "./liner"
 import { resolveFileContent } from "./file"
 import type { EncodeOptions } from "gpt-tokenizer/GptEncoding"
+import { assert } from "./util"
 
 /**
  * Resolves the appropriate token encoder based on the given model ID.
@@ -31,6 +32,7 @@ export async function resolveTokenEncoder(
             decode,
             default: api,
         } = await import(`gpt-tokenizer/model/${module}`)
+        assert(!!encode)
         const { modelName } = api
         const size =
             api.bytePairEncodingCoreProcessor?.mergeableBytePairRankCount +
@@ -50,6 +52,7 @@ export async function resolveTokenEncoder(
             decode,
             default: api,
         } = await import("gpt-tokenizer/model/gpt-4o")
+        assert(!!encode)
         const { modelName, vocabularySize } = api
         return Object.freeze<Tokenizer>({
             model: modelName,