fix long line

gorkemgoknar · Dec 19, 2020 · c3d43c3 · c3d43c3
1 parent 144145e
commit c3d43c3
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 1 deletion.
diff --git a/data/make_persona_file.py b/data/make_persona_file.py
@@ -102,6 +102,12 @@ def get_chat_dialog(dialog):
     if len(name)>22:
       continue
 
+    if len(splitted[1:])> 544:
+      #do not get long lines.
+      #have token limit of 512
+      continue
+
+
     talk = " ".join(splitted[1:]).strip()
 
     talk = preprocess_line(talk)

diff --git a/data/moviepersonafile.json b/data/moviepersonafile.json