[tokenizer] Updates huggingface tokenizer to 0.21.0 (#3572)

* [tokenizer] Updates huggingface tokenizer to 0.21.0 * Update
deepjavalibrary · Jan 3, 2025 · 59f1ea8 · 59f1ea8
1 parent 1f27912
commit 59f1ea8
Show file tree

Hide file tree

Showing 4 changed files with 6 additions and 6 deletions.
diff --git a/android/gradle.properties b/android/gradle.properties
@@ -17,6 +17,6 @@ org.gradle.jvmargs=-Xmx1536m
 android.useAndroidX=true
 # Automatically convert third-party libraries to use AndroidX
 android.enableJetifier=true
-djl_version=0.31.0
+djl_version=0.32.0
 pytorch_version=2.5.1
-tokenizers_version=0.20.3
+tokenizers_version=0.21.0
diff --git a/extensions/tokenizers/rust/Cargo.toml b/extensions/tokenizers/rust/Cargo.toml
@@ -13,7 +13,7 @@ candle-flash-attn = { version = "*", optional = true }
 candle-cublaslt = { git = "https://github.com/huggingface/candle-cublaslt", rev = "cf789b7dd6d4abb19b03b9556442f94f0588b4a0", optional = true }
 candle-layer-norm = { git = "https://github.com/xyang16/candle-layer-norm", rev = "e574de6a7f88bafbede8edf9ee43170c6a8ce51a", optional = true }
 candle-rotary = { git = "https://github.com/huggingface/candle-rotary", rev = "0a718a0856569a92f3112e64f10d07e4447822e8", optional = true }
-tokenizers = { version = "0.20.3" }
+tokenizers = { version = "0.21.0" }
 half = "2.4.0"
 tracing = "0.1.40"
 safetensors = "0.4.3"
@@ -34,7 +34,7 @@ openssl-src = { git = "https://github.com/alexcrichton/openssl-src-rs", rev = "c
 openssl = { version = "0.10", features = ["vendored"] }
 
 [target.'cfg(not(target_os = "android"))'.dependencies]
-tokenizers = { version = "0.20.0", features = ["http"] }
+tokenizers = { version = "0.21.0", features = ["http"] }
 
 [lib]
 crate-type = ["cdylib"]

diff --git a/...ions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizerTest.java b/...ions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizerTest.java
@@ -39,7 +39,7 @@ public class HuggingFaceTokenizerTest {
     public void testVersion() {
         try (HuggingFaceTokenizer tokenizer = HuggingFaceTokenizer.newInstance("bert-base-cased")) {
             String djlVersion = Engine.getDjlVersion();
-            Assert.assertEquals(tokenizer.getVersion(), "0.20.3-" + djlVersion);
+            Assert.assertEquals(tokenizer.getVersion(), "0.21.0-" + djlVersion);
         }
     }
 

diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
@@ -14,7 +14,7 @@ lightgbm = "3.2.110"
 tensorrt = "8.4.1"
 fasttext = "0.9.2"
 sentencepiece = "0.2.0"
-tokenizers = "0.20.3"
+tokenizers = "0.21.0"
 
 slf4j = "2.0.16"
 log4j_slf4j = "2.24.1"