diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..24a8e87 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.png filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore index c8f0442..c3ec037 100644 --- a/.gitignore +++ b/.gitignore @@ -70,3 +70,6 @@ docs/_build/ # Pyenv .python-version + +tmp/ +tools/rwkv_tokenizers_bpe.ipynb diff --git a/Cargo.toml b/Cargo.toml index 3678f53..71d2a5b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,9 @@ name = "rwkv_tokenizer" version = "0.5.2" edition = "2021" authors = ["Cahya Wirawan "] +description = "A fast RWKV Tokenizer" +license = "Apache-2.0" +documentation = "https://github.com/cahya-wirawan/rwkv-tokenizer" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [lib]