Added Llama v2 Model to Textbase with Replicate API, created example bot using Llama, and updated Docs accordingly

README.md

@@ -28,7 +28,7 @@ Coming soon:
 - [x] [PyPI package](https://pypi.org/project/textbase-client/)
 - [x] Easy web deployment via [textbase deploy](/docs/deployment/deploy-from-cli)
 - [ ] SMS integration
-- [ ] Native integration of other models (Claude, Llama, ...)
+- [ ] Native integration of other models (Claude, ...)
 
 ![Demo Deploy GIF](assets/textbase-deploy.gif)
 

docs/docs/examples/llama-bot.md

@@ -0,0 +1,51 @@
+---
+sidebar_position: 3
+---
+
+# Llama bot
+
+This bot makes an API call to Meta's LLama 2 API and processes the user input. It uses Llama-2-7B model.
+
+```py
+from textbase import bot, Message
+from textbase.models import Llama
+from typing import List
+
+# Load your Replicate API key
+Llama.replicate_api_key = "r8_8xsgb2mpX8GB9Iz9BduNR3vwc0vs7Ei1zFi3u"
+
+# Default Prompt for Llama7b. States how the AI Model is supposed to act like
+SYSTEM_PROMPT = """\
+You are a helpful assistant"""
+
+@bot()
+def on_message(message_history: List[Message], state: dict = None):
+
+    # Generate Llama7b responses
+    bot_response = Llama.generate(
+        system_prompt=SYSTEM_PROMPT,
+        message_history=message_history, # Assuming history is the list of user messages
+    )
+
+    response = {
+        "data": {
+            "messages": [
+                {
+                    "data_type": "STRING",
+                    "value": bot_response
+                }
+            ],
+            "state": state
+        },
+        "errors": [
+            {
+                "message": ""
+            }
+        ]
+    }
+
+    return {
+        "status_code": 200,
+        "response": response
+    }
+```

docs/docs/intro.md

@@ -14,4 +14,4 @@ Coming soon:
 - [x] [PyPI package](https://pypi.org/project/textbase-client/)
 - [x] Easy web deployment via [textbase-client deploy](../docs/deployment/deploy-from-cli.md)
 - [ ] SMS integration
-- [ ] Native integration of other models (Claude, Llama, ...)
+- [ ] Native integration of other models (Claude, ...)

examples/llama-bot/main.py

@@ -0,0 +1,41 @@
+from textbase import bot, Message
+from textbase.models import Llama
+from typing import List
+
+# Load your Replicate API key
+Llama.replicate_api_key = "r8_8xsgb2mpX8GB9Iz9BduNR3vwc0vs7Ei1zFi3u"
+
+# Prompt for Llama7b. Llama gets offensive with larger complicated system prompts. This works just fine
+SYSTEM_PROMPT = """\
+You are a helpful assistant"""
+
+@bot()
+def on_message(message_history: List[Message], state: dict = None):
+
+    # Generate Llama7b response by default
+    bot_response = Llama.generate(
+        system_prompt=SYSTEM_PROMPT,
+        message_history=message_history, # Assuming history is the list of user messages
+    )
+
+    response = {
+        "data": {
+            "messages": [
+                {
+                    "data_type": "STRING",
+                    "value": bot_response
+                }
+            ],
+            "state": state
+        },
+        "errors": [
+            {
+                "message": ""
+            }
+        ]
+    }
+
+    return {
+        "status_code": 200,
+        "response": response
+    }

pyproject.toml

@@ -15,6 +15,7 @@ tabulate = "^0.9.0"
 functions-framework = "^3.4.0"
 yaspin = "^3.0.0"
 pydantic = "^2.3.0"
+replicate = "^0.11.0"
 
 [build-system]
 requires = ["poetry-core"]

textbase/models.py

@@ -4,6 +4,7 @@
 import time
 import typing
 import traceback
+import replicate
 
 from textbase import Message
 
@@ -143,4 +144,39 @@ def generate(
         data = json.loads(response.text) # parse the JSON data into a dictionary
         message = data['message']
 
-        return message
+        return message
+
+class Llama:
+
+    @classmethod
+    def generate(
+        cls,
+        system_prompt: str,
+        message_history: list[Message],
+        model = "a16z-infra/llama7b-v2-chat:4f0a4744c7295c024a1de15e1a63c880d3da035fa1f49bfd344fe076074c8eea",
+        temperature = 0.81, #seems to give the best responses
+        top_p = 0.95,
+        max_length = 3008 #same for this
+    ):
+        try:
+            assert cls.replicate_api_key is not None, "Replicate API key is not set."
+            client=replicate.Client(api_token=cls.replicate_api_key) #creating the client for the model
+
+            past_conversation=""
+
+            for message in message_history:
+                if message["role"] == "user":
+                    past_conversation+="[INST] ".join(extract_content_values(message))+" [/INST]\n" #Llama2 paper mentions tokenization of user messages with INST delimiter, otherwise the responses get ugly with time.
+                else:
+                    past_conversation+=" ".join(extract_content_values(message))+"\n" #response messages dont need any delimiters
+
+            response=client.run(model,input={"prompt": past_conversation,"system_prompt": system_prompt,"temperature":temperature,"top_p":top_p,"max_length":max_length,"repetition_penalty":1})
+
+            resp=""
+            for word in response:
+                resp+=word
+
+            return resp
+
+        except Exception:
+            print(f"An exception occured while using this model, please try using another model.\nException: {traceback.format_exc()}.")