From 8e0a0ff0b47d51b8bed69a61e0f3b00507982762 Mon Sep 17 00:00:00 2001
From: deedy5 <65482418+deedy5@users.noreply.github.com>
Date: Thu, 25 Jul 2024 00:07:49 +0300
Subject: [PATCH] chat(): bugfix token counting, rename
 `_chat_conversation_size` to `_chat_tokens_count`

---
 duckduckgo_search/cli.py               | 2 +-
 duckduckgo_search/duckduckgo_search.py | 8 +++++---
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/duckduckgo_search/cli.py b/duckduckgo_search/cli.py
index 0483926..e9ad7dc 100644
--- a/duckduckgo_search/cli.py
+++ b/duckduckgo_search/cli.py
@@ -165,7 +165,7 @@ def chat(load, proxy, multiline, timeout, model):
             client._chat_messages = cache.get("messages", [])
 
     while True:
-        print(f"{'-'*78}\nYou[{model=} contextsize={client._chat_conversation_size}]: ", end="")
+        print(f"{'-'*78}\nYou[{model=} tokens={client._chat_tokens_count}]: ", end="")
         if multiline:
             print(f"""[multiline, send message: ctrl+{"Z" if sys.platform == "win32" else "D"}]""")
             user_input = sys.stdin.read()
diff --git a/duckduckgo_search/duckduckgo_search.py b/duckduckgo_search/duckduckgo_search.py
index 78ed4d0..4cfe296 100644
--- a/duckduckgo_search/duckduckgo_search.py
+++ b/duckduckgo_search/duckduckgo_search.py
@@ -81,8 +81,8 @@ def __init__(
             verify=False,
         )
         self._exception_event = Event()
-        self._chat_conversation_size = 0
         self._chat_messages: List[Dict[str, str]] = []
+        self._chat_tokens_count = 0
         self._chat_vqd: str = ""
 
     def __enter__(self) -> "DDGS":
@@ -155,6 +155,7 @@ def chat(self, keywords: str, model: str = "gpt-3.5", timeout: int = 20) -> str:
             self._chat_vqd = resp.headers.get("x-vqd-4", "")
 
         self._chat_messages.append({"role": "user", "content": keywords})
+        self._chat_tokens_count += len(keywords) // 4 if len(keywords) >= 4 else 1  # approximate number of tokens
 
         json_data = {
             "model": models[model],
@@ -182,11 +183,12 @@ def chat(self, keywords: str, model: str = "gpt-3.5", timeout: int = 20) -> str:
                         else RatelimitException(err_message)
                     )
                 raise DuckDuckGoSearchException(err_message)
-            results.append(x.get("message", ""))
+            elif message := x.get("message"):
+                results.append(message)
         result = "".join(results)
 
         self._chat_messages.append({"role": "assistant", "content": result})
-        self._chat_conversation_size = sum(len(d["content"]) for d in self._chat_messages)
+        self._chat_tokens_count += len(results)
         return result
 
     def text(