From 7fbd177435312f73c1e38cfc9b13826be4acf0fd Mon Sep 17 00:00:00 2001 From: Bearice Ren Date: Wed, 15 May 2024 10:06:54 +0900 Subject: [PATCH] update tiktoken for gpt-4o --- package-lock.json | 6 +++--- utils/server/tiktoken.ts | 5 ++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index 9bd6613f40..fada3d1717 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8931,9 +8931,9 @@ } }, "node_modules/tiktoken": { - "version": "1.0.13", - "resolved": "https://registry.npmjs.org/tiktoken/-/tiktoken-1.0.13.tgz", - "integrity": "sha512-JaL9ZnvTbGFMDIBeGdVkLt4qWTeCPw+n7Ock+wceAGRenuHA6nOOvMJFliNDyXsjg2osGKJWsXtO2xc74VxyDw==" + "version": "1.0.15", + "resolved": "https://registry.npmjs.org/tiktoken/-/tiktoken-1.0.15.tgz", + "integrity": "sha512-sCsrq/vMWUSEW29CJLNmPvWxlVp7yh2tlkAjpJltIKqp5CKf98ZNpdeHRmAlPVFlGEbswDc6SmI8vz64W/qErw==" }, "node_modules/time-zone": { "version": "1.0.0", diff --git a/utils/server/tiktoken.ts b/utils/server/tiktoken.ts index bd4b39054e..1fe9811b95 100644 --- a/utils/server/tiktoken.ts +++ b/utils/server/tiktoken.ts @@ -1,7 +1,7 @@ import cl100k from 'tiktoken/encoders/cl100k_base.json'; import p50k from 'tiktoken/encoders/p50k_base.json'; import { Tiktoken } from 'tiktoken/lite'; -const { encoding_for_model } = require('tiktoken'); +import { encoding_for_model } from 'tiktoken'; export const getTiktokenEncoding = async (model: string): Promise => { // Azure fix @@ -9,6 +9,9 @@ export const getTiktokenEncoding = async (model: string): Promise => { if (modelId.indexOf('text-davinci-') !== -1) { return new Tiktoken(p50k.bpe_ranks, p50k.special_tokens, p50k.pat_str); } + if (modelId.indexOf('gpt-4o') !== -1) { + return encoding_for_model('gpt-4o'); + } if (modelId.indexOf('gpt-3.5') !== -1) { return encoding_for_model('gpt-3.5-turbo', { '<|im_start|>': 100264,