fix: path

AI21Labs · Mar 26, 2024 · 7e4c8a9 · 7e4c8a9
1 parent 7e37fd0
commit 7e4c8a9
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/ai21_tokenizer/tokenizer_factory.py b/ai21_tokenizer/tokenizer_factory.py
@@ -30,7 +30,7 @@ def get_tokenizer(cls, tokenizer_name: str = PreTrainedTokenizers.J2_TOKENIZER)
 
         if tokenizer_name == PreTrainedTokenizers.JAMBA_TOKENIZER:
             os.environ["TRANSFORMERS_NO_ADVISORY_WARNINGS"] = "true"  # Disable Huggingface advice warning
-            return JambaTokenizer(model_path=model_path.parent, cache_dir=_MODEL_CACHE_DIR)
+            return JambaTokenizer(model_path=model_path, cache_dir=_MODEL_CACHE_DIR)
 
         config = cls._get_config(tokenizer_name)
         return JurassicTokenizer(model_path=model_path, config=config)