v0.7.3

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "pyrwkv_tokenizer"
-version = "0.7.2"
+version = "0.7.3"
 edition = "2021"
 authors = ["Cahya Wirawan <cahya.wirawan@gmail.com>"]
 description = "A fast RWKV Tokenizer"
@@ -22,4 +22,4 @@ crate-type = ["cdylib"]
 
 [dependencies]
 pyo3 = "0.21.2"
-rwkv-tokenizer = "0.7.1"
+rwkv-tokenizer = "0.7.3"

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "pyrwkv_tokenizer"
-version = "0.7.2"
+version = "0.7.3"
 requires-python = ">=3.8"
 description = "RWKV Tokenizer"
 readme = "README.md"

pyrwkv_tokenizer/__init__.py

@@ -1,19 +1,22 @@
 # import the contents of the Rust library into the Python extension
 from .pyrwkv_tokenizer import *
 from .pyrwkv_tokenizer import __all__
+from pathlib import Path
 
 # optional: include the documentation from the Rust module
 from .pyrwkv_tokenizer import __doc__  # noqa: F401
 
 __all__ = __all__ + ["RWKVTokenizer"]
-__version__ = "0.7.2"
+__version__ = "0.7.3"
 
 
 class RWKVTokenizer:
-    def __init__(self, name="WorldTokenizer") -> None:
+    def __init__(self, name="WorldTokenizer", vocab_filepath=None) -> None:
         if name != "WorldTokenizer":
             raise Exception(f"The {name} is not supported.")
-        self.tokenizer = WorldTokenizer()
+        self.vocab_filepath = str(Path(__path__[0]) / "rwkv_vocab_v20230424.txt") \
+            if vocab_filepath is None else vocab_filepath
+        self.tokenizer = WorldTokenizer(self.vocab_filepath)
 
     def encode(self, text: str):
         tokens_ids = self.tokenizer.encode(text)