BUG: Asking to pad but the tokenizer does not have a padding token (#407

)
xorbitsai · Aug 30, 2023 · 60c0be5 · 60c0be5
1 parent 4a23d57
commit 60c0be5
Showing 1 changed file with 6 additions and 0 deletions.
diff --git a/xinference/model/llm/pytorch/llama_2.py b/xinference/model/llm/pytorch/llama_2.py
@@ -39,6 +39,9 @@ def __init__(
 
     def _load_model(self, kwargs: dict):
         model, tokenizer = super()._load_model(kwargs)
+        # Llama has no pad token by default
+        # https://github.com/huggingface/transformers/blob/07998ef39926b76d3f6667025535d0859eed61c3/docs/source/en/llm_tutorial.md?plain=1#L125
+        tokenizer.pad_token = tokenizer.eos_token
         model.config.eos_token_id = tokenizer.eos_token_id
         model.config.pad_token_id = tokenizer.pad_token_id
         return model, tokenizer
@@ -76,6 +79,9 @@ def __init__(
 
     def _load_model(self, kwargs: dict):
         model, tokenizer = super()._load_model(kwargs)
+        # Llama has no pad token by default
+        # https://github.com/huggingface/transformers/blob/07998ef39926b76d3f6667025535d0859eed61c3/docs/source/en/llm_tutorial.md?plain=1#L125
+        tokenizer.pad_token = tokenizer.eos_token
         model.config.eos_token_id = tokenizer.eos_token_id
         model.config.pad_token_id = tokenizer.pad_token_id
         return model, tokenizer