Add error logging for encoding failures in token estimation function

microsoft · Sep 25, 2024 · 981d1dd · 981d1dd
1 parent 37b8a38
commit 981d1dd
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/packages/core/src/tokens.ts b/packages/core/src/tokens.ts
@@ -17,6 +17,8 @@ export function estimateTokens(text: string, encoder: TokenEncoder) {
         // Return the length of the encoded text plus a constant overhead
         return encoder(text).length + ESTIMATE_TOKEN_OVERHEAD
     } catch (e) {
+        // If encoding fails, log the error in verbose mode
+        logVerbose(e)
         // Fallback: Estimate token count as one-fourth of text length plus overhead
         // This provides a rough estimate in case of encoding errors
         return (text.length >> 2) + ESTIMATE_TOKEN_OVERHEAD