update tiktoken for gpt-4o

iknow · May 15, 2024 · 7fbd177 · 7fbd177
1 parent fb1d837
commit 7fbd177
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 4 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/utils/server/tiktoken.ts b/utils/server/tiktoken.ts
@@ -1,14 +1,17 @@
 import cl100k from 'tiktoken/encoders/cl100k_base.json';
 import p50k from 'tiktoken/encoders/p50k_base.json';
 import { Tiktoken } from 'tiktoken/lite';
-const { encoding_for_model } = require('tiktoken');
+import { encoding_for_model } from 'tiktoken';
 
 export const getTiktokenEncoding = async (model: string): Promise<Tiktoken> => {
   // Azure fix
   const modelId = model.replace('gpt-35', 'gpt-3.5')
   if (modelId.indexOf('text-davinci-') !== -1) {
     return new Tiktoken(p50k.bpe_ranks, p50k.special_tokens, p50k.pat_str);
   }
+  if (modelId.indexOf('gpt-4o') !== -1) {
+    return encoding_for_model('gpt-4o');
+  }
   if (modelId.indexOf('gpt-3.5') !== -1) {
     return encoding_for_model('gpt-3.5-turbo', {
       '<|im_start|>': 100264,