refactor: Added import to init

AI21Labs · Mar 26, 2024 · 7e37fd0 · 7e37fd0
1 parent 7dba113
commit 7e37fd0
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/ai21_tokenizer/__init__.py b/ai21_tokenizer/__init__.py
@@ -1,8 +1,8 @@
 from ai21_tokenizer.base_tokenizer import BaseTokenizer
 from ai21_tokenizer.jurassic_tokenizer import JurassicTokenizer
-from ai21_tokenizer.tokenizer_factory import TokenizerFactory as Tokenizer
+from ai21_tokenizer.tokenizer_factory import TokenizerFactory as Tokenizer, PreTrainedTokenizers
 from .version import VERSION
 
 __version__ = VERSION
 
-__all__ = ["Tokenizer", "JurassicTokenizer", "BaseTokenizer", "__version__"]
+__all__ = ["Tokenizer", "JurassicTokenizer", "BaseTokenizer", "__version__", "PreTrainedTokenizers"]