From 981d1dd691a4511553dd303f448d61af664ddf19 Mon Sep 17 00:00:00 2001
From: Peli de Halleux <pelikhan@users.noreply.github.com>
Date: Wed, 25 Sep 2024 19:15:56 +0000
Subject: [PATCH] Add error logging for encoding failures in token estimation
 function

---
 packages/core/src/tokens.ts | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/packages/core/src/tokens.ts b/packages/core/src/tokens.ts
index ae81b78e4f..d48bff8f75 100644
--- a/packages/core/src/tokens.ts
+++ b/packages/core/src/tokens.ts
@@ -17,6 +17,8 @@ export function estimateTokens(text: string, encoder: TokenEncoder) {
         // Return the length of the encoded text plus a constant overhead
         return encoder(text).length + ESTIMATE_TOKEN_OVERHEAD
     } catch (e) {
+        // If encoding fails, log the error in verbose mode
+        logVerbose(e)
         // Fallback: Estimate token count as one-fourth of text length plus overhead
         // This provides a rough estimate in case of encoding errors
         return (text.length >> 2) + ESTIMATE_TOKEN_OVERHEAD