chat(): bugfix token counting, rename _chat_conversation_size to `_…

…chat_tokens_count`

duckduckgo_search/cli.py

@@ -165,7 +165,7 @@ def chat(load, proxy, multiline, timeout, model):
             client._chat_messages = cache.get("messages", [])
 
     while True:
-        print(f"{'-'*78}\nYou[{model=} contextsize={client._chat_conversation_size}]: ", end="")
+        print(f"{'-'*78}\nYou[{model=} tokens={client._chat_tokens_count}]: ", end="")
         if multiline:
             print(f"""[multiline, send message: ctrl+{"Z" if sys.platform == "win32" else "D"}]""")
             user_input = sys.stdin.read()

duckduckgo_search/duckduckgo_search.py

@@ -81,8 +81,8 @@ def __init__(
             verify=False,
         )
         self._exception_event = Event()
-        self._chat_conversation_size = 0
         self._chat_messages: List[Dict[str, str]] = []
+        self._chat_tokens_count = 0
         self._chat_vqd: str = ""
 
     def __enter__(self) -> "DDGS":
@@ -155,6 +155,7 @@ def chat(self, keywords: str, model: str = "gpt-3.5", timeout: int = 20) -> str:
             self._chat_vqd = resp.headers.get("x-vqd-4", "")
 
         self._chat_messages.append({"role": "user", "content": keywords})
+        self._chat_tokens_count += len(keywords) // 4 if len(keywords) >= 4 else 1  # approximate number of tokens
 
         json_data = {
             "model": models[model],
@@ -182,11 +183,12 @@ def chat(self, keywords: str, model: str = "gpt-3.5", timeout: int = 20) -> str:
                         else RatelimitException(err_message)
                     )
                 raise DuckDuckGoSearchException(err_message)
-            results.append(x.get("message", ""))
+            elif message := x.get("message"):
+                results.append(message)
         result = "".join(results)
 
         self._chat_messages.append({"role": "assistant", "content": result})
-        self._chat_conversation_size = sum(len(d["content"]) for d in self._chat_messages)
+        self._chat_tokens_count += len(results)
         return result
 
     def text(