diff --git a/extensions/tokenizers/rust/Cargo.toml b/extensions/tokenizers/rust/Cargo.toml index fe9346fad9d..93394982d82 100644 --- a/extensions/tokenizers/rust/Cargo.toml +++ b/extensions/tokenizers/rust/Cargo.toml @@ -13,7 +13,7 @@ candle-flash-attn = { version = "*", optional = true } candle-cublaslt = { git = "https://github.com/huggingface/candle-cublaslt", rev = "cf789b7dd6d4abb19b03b9556442f94f0588b4a0", optional = true } candle-layer-norm = { git = "https://github.com/xyang16/candle-layer-norm", rev = "e574de6a7f88bafbede8edf9ee43170c6a8ce51a", optional = true } candle-rotary = { git = "https://github.com/huggingface/candle-rotary", rev = "0a718a0856569a92f3112e64f10d07e4447822e8", optional = true } -tokenizers = { version = "0.20.0", features = ["http"] } +tokenizers = { version = "0.20.3", features = ["http"] } half = "2.4.0" tracing = "0.1.40" safetensors = "0.4.3" diff --git a/extensions/tokenizers/rust/src/lib.rs b/extensions/tokenizers/rust/src/lib.rs index 2c12559dd6a..59bc0a4f2f4 100644 --- a/extensions/tokenizers/rust/src/lib.rs +++ b/extensions/tokenizers/rust/src/lib.rs @@ -59,7 +59,7 @@ pub extern "system" fn Java_ai_djl_huggingface_tokenizers_jni_TokenizersLibrary_ let mut parameters = FromPretrainedParameters::default(); if !hf_token.is_null() { let hf_token: String = env.get_string(&hf_token).unwrap().into(); - parameters.auth_token = Some(hf_token); + parameters.token = Some(hf_token); } let tokenizer = Tokenizer::from_pretrained(identifier, Some(parameters));