[tokenizer] Adds getters for HuggingfaceTokenizer

extensions/tokenizers/src/main/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizer.java

@@ -449,6 +449,53 @@ public void enableBatch() {
         }
     }
 
+    /**
+     * Returns the truncation policy.
+     *
+     * @return the truncation policy
+     */
+    public String getTruncation() {
+        return truncation.name();
+    }
+
+    /**
+     * Returns the padding policy.
+     *
+     * @return the padding policy
+     */
+    public String getPadding() {
+        return padding.name();
+    }
+
+    /**
+     * Returns the max token length.
+     *
+     * @return the max token length
+     */
+    public int getMaxLength() {
+        return maxLength;
+    }
+
+    /**
+     * Returns the stride to use in overflow overlap when truncating sequences longer than the model
+     * supports.
+     *
+     * @return the stride to use in overflow overlap when truncating sequences longer than the model
+     *     supports
+     */
+    public int getStride() {
+        return stride;
+    }
+
+    /**
+     * Returns the padToMultipleOf for padding.
+     *
+     * @return the padToMultipleOf for padding
+     */
+    public int getPadToMultipleOf() {
+        return padToMultipleOf;
+    }
+
     /**
      * Creates a builder to build a {@code HuggingFaceTokenizer}.
      *

extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizerTest.java

@@ -41,6 +41,12 @@ public void testTokenizer() throws IOException {
         };
 
         try (HuggingFaceTokenizer tokenizer = HuggingFaceTokenizer.newInstance("bert-base-cased")) {
+            Assert.assertEquals(tokenizer.getTruncation(), "DO_NOT_TRUNCATE");
+            Assert.assertEquals(tokenizer.getPadding(), "DO_NOT_PAD");
+            Assert.assertEquals(tokenizer.getMaxLength(), -1);
+            Assert.assertEquals(tokenizer.getStride(), 0);
+            Assert.assertEquals(tokenizer.getPadToMultipleOf(), 0);
+
             List<String> ret = tokenizer.tokenize(input);
             Assert.assertEquals(ret.toArray(Utils.EMPTY_ARRAY), expected);
             Encoding encoding = tokenizer.encode(input);