BUG: llama_cpp model context length (#429)

xorbitsai · Sep 6, 2023 · 4ffc1b7 · 4ffc1b7
1 parent 931aa6b
commit 4ffc1b7
Showing 1 changed file with 2 additions and 5 deletions.
diff --git a/xinference/model/llm/ggml/llamacpp.py b/xinference/model/llm/ggml/llamacpp.py
@@ -14,7 +14,6 @@
 
 import logging
 import os
-import platform
 from typing import TYPE_CHECKING, Iterator, List, Optional, TypedDict, Union
 
 from ....types import (
@@ -123,11 +122,9 @@ def _sanitize_model_config(
     ) -> LlamaCppModelConfig:
         if llamacpp_model_config is None:
             llamacpp_model_config = LlamaCppModelConfig()
-        if platform.system() == "Windows":
-            llamacpp_model_config.setdefault("n_ctx", 512)
-        else:
-            llamacpp_model_config.setdefault("n_ctx", 2048)
 
+        if self.model_family.context_length:
+            llamacpp_model_config.setdefault("n_ctx", self.model_family.context_length)
         llamacpp_model_config.setdefault("embedding", True)
         llamacpp_model_config.setdefault("use_mmap", False)
         llamacpp_model_config.setdefault("use_mlock", True)