Merge branch 'main' into add-is-start-to-encode

AI21Labs · Dec 28, 2023 · 4954a72 · 4954a72
2 parents 25d6688 + 96f384f
commit 4954a72
Show file tree

Hide file tree

Showing 3 changed files with 30 additions and 2 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,8 +2,35 @@
 
 
 
+## v0.5.0 (2023-12-28)
+
+### Feature
+
+* feat: Add more special tokens (#71)
+
+* fix: commitizen tag starts with &#34;v&#34;
+
+* feat: add eos_id
+
+* feat: Add newline_id
+
+* fix: typo &#34;_newline_piece&#34; instead of &#34;newline_piece&#34;
+
+* fix: newline_id already existed as &#34;private&#34;. Just make it &#34;public&#34;
+
+* fix: forgot to rename everywhere ([`9a9e1a8`](https://github.com/AI21Labs/ai21-tokenizer/commit/9a9e1a8a21dbfd4d3f983ce8ec1f97335470e2c6))
+
+### Fix
+
+* fix: commitizen tag starts with &#34;v&#34; (#70) ([`cf495ad`](https://github.com/AI21Labs/ai21-tokenizer/commit/cf495ada8341131c0d022f02d2c9e86cee8723ba))
+
+
 ## v0.4.0 (2023-12-28)
 
+### Chore
+
+* chore(release): v0.4.0 [skip ci] ([`b761edc`](https://github.com/AI21Labs/ai21-tokenizer/commit/b761edc0947b132192d4f963dd8d8704df4ebac4))
+
 ### Feature
 
 * feat: add pad_id and bos_id to jurassic_tokenizer (#69) ([`ffb2ce3`](https://github.com/AI21Labs/ai21-tokenizer/commit/ffb2ce38aa59eec03305c4fdc6bfedd99c7b5255))

diff --git a/ai21_tokenizer/version.py b/ai21_tokenizer/version.py
@@ -1 +1 @@
-VERSION = "0.4.0"
+VERSION = "0.5.0"
diff --git a/pyproject.toml b/pyproject.toml
@@ -59,7 +59,7 @@ schema_pattern = "(build|ci|docs|feat|fix|perf|refactor|style|test|chore|revert|
 
 [tool.poetry]
 name = "ai21-tokenizer"
-version = "0.4.0"
+version = "0.5.0"
 description = ""
 authors = ["AI21 Labs"]
 readme = "README.md"
@@ -109,6 +109,7 @@ newline_sequence = "\n"
 
 
 
+
 [tool.poetry.group.test.dependencies]
 coverage = "^7.1.0"
 pytest = "7.2.1"