Move io and sentencepiece packages back to tensorflow

- actually sentencepiece does use TensorFlow to load the SP model so it must stay in tensorflow package - io is also mostly used in loading TensorFlow models
JohnSnowLabs · Feb 17, 2023 · 2f41ea6 · 2f41ea6
1 parent 5a7fc84
commit 2f41ea6
Show file tree

Hide file tree

Showing 31 changed files with 49 additions and 49 deletions.
diff --git a/src/main/scala/com/johnsnowlabs/ml/ai/AlbertClassification.scala b/src/main/scala/com/johnsnowlabs/ml/ai/AlbertClassification.scala
@@ -16,7 +16,7 @@
 
 package com.johnsnowlabs.ml.ai
 
-import com.johnsnowlabs.ml.ai.util.sentencepiece.{SentencePieceWrapper, SentencepieceEncoder}
+import com.johnsnowlabs.ml.tensorflow.sentencepiece.{SentencePieceWrapper, SentencepieceEncoder}
 import com.johnsnowlabs.ml.tensorflow.sign.{ModelSignatureConstants, ModelSignatureManager}
 import com.johnsnowlabs.ml.tensorflow.{TensorResources, TensorflowWrapper}
 import com.johnsnowlabs.nlp.annotators.common._

diff --git a/src/main/scala/com/johnsnowlabs/ml/ai/CamemBertClassification.scala b/src/main/scala/com/johnsnowlabs/ml/ai/CamemBertClassification.scala
@@ -16,7 +16,7 @@
 
 package com.johnsnowlabs.ml.ai
 
-import com.johnsnowlabs.ml.ai.util.sentencepiece.{SentencePieceWrapper, SentencepieceEncoder}
+import com.johnsnowlabs.ml.tensorflow.sentencepiece.{SentencePieceWrapper, SentencepieceEncoder}
 import com.johnsnowlabs.ml.tensorflow.sign.{ModelSignatureConstants, ModelSignatureManager}
 import com.johnsnowlabs.ml.tensorflow.{TensorResources, TensorflowWrapper}
 import com.johnsnowlabs.nlp.annotators.common._

diff --git a/src/main/scala/com/johnsnowlabs/ml/ai/DeBertaClassification.scala b/src/main/scala/com/johnsnowlabs/ml/ai/DeBertaClassification.scala
@@ -16,7 +16,7 @@
 
 package com.johnsnowlabs.ml.ai
 
-import com.johnsnowlabs.ml.ai.util.sentencepiece.{SentencePieceWrapper, SentencepieceEncoder}
+import com.johnsnowlabs.ml.tensorflow.sentencepiece.{SentencePieceWrapper, SentencepieceEncoder}
 import com.johnsnowlabs.ml.tensorflow.sign.{ModelSignatureConstants, ModelSignatureManager}
 import com.johnsnowlabs.ml.tensorflow.{TensorResources, TensorflowWrapper}
 import com.johnsnowlabs.nlp.annotators.common._

diff --git a/src/main/scala/com/johnsnowlabs/ml/ai/XlmRoBertaClassification.scala b/src/main/scala/com/johnsnowlabs/ml/ai/XlmRoBertaClassification.scala
@@ -16,7 +16,7 @@
 
 package com.johnsnowlabs.ml.ai
 
-import com.johnsnowlabs.ml.ai.util.sentencepiece.{SentencePieceWrapper, SentencepieceEncoder}
+import com.johnsnowlabs.ml.tensorflow.sentencepiece.{SentencePieceWrapper, SentencepieceEncoder}
 import com.johnsnowlabs.ml.tensorflow.sign.{ModelSignatureConstants, ModelSignatureManager}
 import com.johnsnowlabs.ml.tensorflow.{TensorResources, TensorflowWrapper}
 import com.johnsnowlabs.nlp.annotators.common._

diff --git a/src/main/scala/com/johnsnowlabs/ml/ai/XlnetClassification.scala b/src/main/scala/com/johnsnowlabs/ml/ai/XlnetClassification.scala
@@ -16,7 +16,7 @@
 
 package com.johnsnowlabs.ml.ai
 
-import com.johnsnowlabs.ml.ai.util.sentencepiece.{SentencePieceWrapper, SentencepieceEncoder}
+import com.johnsnowlabs.ml.tensorflow.sentencepiece.{SentencePieceWrapper, SentencepieceEncoder}
 import com.johnsnowlabs.ml.tensorflow.sign.{ModelSignatureConstants, ModelSignatureManager}
 import com.johnsnowlabs.ml.tensorflow.{TensorResources, TensorflowWrapper}
 import com.johnsnowlabs.nlp.annotators.common._

diff --git a/src/main/scala/com/johnsnowlabs/ml/tensorflow/TensorflowSerializeModel.scala b/src/main/scala/com/johnsnowlabs/ml/tensorflow/TensorflowSerializeModel.scala
@@ -16,7 +16,7 @@
 
 package com.johnsnowlabs.ml.tensorflow
 
-import com.johnsnowlabs.ml.ai.util.sentencepiece.LoadSentencepiece
+import com.johnsnowlabs.ml.tensorflow.sentencepiece.LoadSentencepiece
 import com.johnsnowlabs.nlp.annotators.ner.dl.LoadsContrib
 import com.johnsnowlabs.util.FileHelper
 import org.apache.commons.io.FileUtils

diff --git a/src/main/scala/com/johnsnowlabs/ml/tensorflow/TensorflowWrapper.scala b/src/main/scala/com/johnsnowlabs/ml/tensorflow/TensorflowWrapper.scala
@@ -16,8 +16,8 @@
 
 package com.johnsnowlabs.ml.tensorflow
 
-import com.johnsnowlabs.ml.ai.util.io.ChunkBytes
-import com.johnsnowlabs.ml.ai.util.sentencepiece.LoadSentencepiece
+import com.johnsnowlabs.ml.tensorflow.io.ChunkBytes
+import com.johnsnowlabs.ml.tensorflow.sentencepiece.LoadSentencepiece
 import com.johnsnowlabs.ml.tensorflow.sign.ModelSignatureManager
 import com.johnsnowlabs.nlp.annotators.ner.dl.LoadsContrib
 import com.johnsnowlabs.nlp.util.io.ResourceHelper

diff --git a/...hnsnowlabs/ml/ai/util/io/ChunkBytes.scala → ...nowlabs/ml/tensorflow/io/ChunkBytes.scala b/...hnsnowlabs/ml/ai/util/io/ChunkBytes.scala → ...nowlabs/ml/tensorflow/io/ChunkBytes.scala
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package com.johnsnowlabs.ml.ai.util.io
+package com.johnsnowlabs.ml.tensorflow.io
 
 import java.io.{BufferedOutputStream, FileInputStream, FileOutputStream, IOException}
 import java.nio.file.{Files, Path}

diff --git a/...til/sentencepiece/LoadSentencepiece.scala → ...low/sentencepiece/LoadSentencepiece.scala b/...til/sentencepiece/LoadSentencepiece.scala → ...low/sentencepiece/LoadSentencepiece.scala
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package com.johnsnowlabs.ml.ai.util.sentencepiece
+package com.johnsnowlabs.ml.tensorflow.sentencepiece
 
 import com.johnsnowlabs.nlp.util.io.ResourceHelper
 import org.apache.commons.lang.SystemUtils

diff --git a/...i/util/sentencepiece/NativeLibLoader.java → ...orflow/sentencepiece/NativeLibLoader.java b/...i/util/sentencepiece/NativeLibLoader.java → ...orflow/sentencepiece/NativeLibLoader.java
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package com.johnsnowlabs.ml.ai.util.sentencepiece;
+package com.johnsnowlabs.ml.tensorflow.sentencepiece;
 
 import java.io.*;
 

diff --git a/...sentencepiece/SentencePieceException.java → ...sentencepiece/SentencePieceException.java b/...sentencepiece/SentencePieceException.java → ...sentencepiece/SentencePieceException.java
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package com.johnsnowlabs.ml.ai.util.sentencepiece;
+package com.johnsnowlabs.ml.tensorflow.sentencepiece;
 
 public class SentencePieceException extends RuntimeException {
 

diff --git a/.../util/sentencepiece/SentencePieceJNI.java → ...rflow/sentencepiece/SentencePieceJNI.java b/.../util/sentencepiece/SentencePieceJNI.java → ...rflow/sentencepiece/SentencePieceJNI.java
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package com.johnsnowlabs.ml.ai.util.sentencepiece;
+package com.johnsnowlabs.ml.tensorflow.sentencepiece;
 
 import java.io.IOException;
 

diff --git a/...sentencepiece/SentencePieceProcessor.java → ...sentencepiece/SentencePieceProcessor.java b/...sentencepiece/SentencePieceProcessor.java → ...sentencepiece/SentencePieceProcessor.java
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package com.johnsnowlabs.ml.ai.util.sentencepiece;
+package com.johnsnowlabs.ml.tensorflow.sentencepiece;
 
 import java.util.Arrays;
 import java.util.List;

diff --git a/.../sentencepiece/SentencePieceWrapper.scala → .../sentencepiece/SentencePieceWrapper.scala b/.../sentencepiece/SentencePieceWrapper.scala → .../sentencepiece/SentencePieceWrapper.scala
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package com.johnsnowlabs.ml.ai.util.sentencepiece
+package com.johnsnowlabs.ml.tensorflow.sentencepiece
 
 import org.apache.commons.io.FileUtils
 import org.apache.hadoop.fs.{FileSystem, Path}

diff --git a/.../sentencepiece/SentencepieceEncoder.scala → .../sentencepiece/SentencepieceEncoder.scala b/.../sentencepiece/SentencepieceEncoder.scala → .../sentencepiece/SentencepieceEncoder.scala
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package com.johnsnowlabs.ml.ai.util.sentencepiece
+package com.johnsnowlabs.ml.tensorflow.sentencepiece
 
 import com.johnsnowlabs.nlp.annotators.common.{IndexedToken, Sentence, TokenPiece}
 

diff --git a/src/main/scala/com/johnsnowlabs/ml/util/LoadExternalModel.scala b/src/main/scala/com/johnsnowlabs/ml/util/LoadExternalModel.scala
@@ -16,7 +16,7 @@
 
 package com.johnsnowlabs.ml.util
 
-import com.johnsnowlabs.ml.ai.util.sentencepiece.SentencePieceWrapper
+import com.johnsnowlabs.ml.tensorflow.sentencepiece.SentencePieceWrapper
 import com.johnsnowlabs.nlp.util.io.{ExternalResource, ReadAs, ResourceHelper}
 
 import java.io.File

diff --git a/...main/scala/com/johnsnowlabs/nlp/annotators/classifier/dl/AlbertForQuestionAnswering.scala b/...main/scala/com/johnsnowlabs/nlp/annotators/classifier/dl/AlbertForQuestionAnswering.scala
@@ -16,13 +16,13 @@
 
 package com.johnsnowlabs.nlp.annotators.classifier.dl
 
-import com.johnsnowlabs.ml.ai.util.sentencepiece.{
+import com.johnsnowlabs.ml.ai.{AlbertClassification, MergeTokenStrategy}
+import com.johnsnowlabs.ml.tensorflow._
+import com.johnsnowlabs.ml.tensorflow.sentencepiece.{
   ReadSentencePieceModel,
   SentencePieceWrapper,
   WriteSentencePieceModel
 }
-import com.johnsnowlabs.ml.ai.{AlbertClassification, MergeTokenStrategy}
-import com.johnsnowlabs.ml.tensorflow._
 import com.johnsnowlabs.ml.util.LoadExternalModel.{
   loadSentencePieceAsset,
   modelSanityCheck,

diff --git a/...scala/com/johnsnowlabs/nlp/annotators/classifier/dl/AlbertForSequenceClassification.scala b/...scala/com/johnsnowlabs/nlp/annotators/classifier/dl/AlbertForSequenceClassification.scala
@@ -17,12 +17,12 @@
 package com.johnsnowlabs.nlp.annotators.classifier.dl
 
 import com.johnsnowlabs.ml.ai.AlbertClassification
-import com.johnsnowlabs.ml.ai.util.sentencepiece.{
+import com.johnsnowlabs.ml.tensorflow._
+import com.johnsnowlabs.ml.tensorflow.sentencepiece.{
   ReadSentencePieceModel,
   SentencePieceWrapper,
   WriteSentencePieceModel
 }
-import com.johnsnowlabs.ml.tensorflow._
 import com.johnsnowlabs.ml.util.LoadExternalModel.{
   loadSentencePieceAsset,
   loadTextAsset,

diff --git a/...in/scala/com/johnsnowlabs/nlp/annotators/classifier/dl/AlbertForTokenClassification.scala b/...in/scala/com/johnsnowlabs/nlp/annotators/classifier/dl/AlbertForTokenClassification.scala
@@ -17,12 +17,12 @@
 package com.johnsnowlabs.nlp.annotators.classifier.dl
 
 import com.johnsnowlabs.ml.ai.AlbertClassification
-import com.johnsnowlabs.ml.ai.util.sentencepiece.{
+import com.johnsnowlabs.ml.tensorflow._
+import com.johnsnowlabs.ml.tensorflow.sentencepiece.{
   ReadSentencePieceModel,
   SentencePieceWrapper,
   WriteSentencePieceModel
 }
-import com.johnsnowlabs.ml.tensorflow._
 import com.johnsnowlabs.ml.util.LoadExternalModel.{
   loadSentencePieceAsset,
   loadTextAsset,

diff --git a/...n/scala/com/johnsnowlabs/nlp/annotators/classifier/dl/CamemBertForQuestionAnswering.scala b/...n/scala/com/johnsnowlabs/nlp/annotators/classifier/dl/CamemBertForQuestionAnswering.scala
@@ -17,12 +17,12 @@
 package com.johnsnowlabs.nlp.annotators.classifier.dl
 
 import com.johnsnowlabs.ml.ai._
-import com.johnsnowlabs.ml.ai.util.sentencepiece.{
+import com.johnsnowlabs.ml.tensorflow._
+import com.johnsnowlabs.ml.tensorflow.sentencepiece.{
   ReadSentencePieceModel,
   SentencePieceWrapper,
   WriteSentencePieceModel
 }
-import com.johnsnowlabs.ml.tensorflow._
 import com.johnsnowlabs.ml.util.LoadExternalModel.{
   loadSentencePieceAsset,
   modelSanityCheck,

diff --git a/...la/com/johnsnowlabs/nlp/annotators/classifier/dl/CamemBertForSequenceClassification.scala b/...la/com/johnsnowlabs/nlp/annotators/classifier/dl/CamemBertForSequenceClassification.scala
@@ -17,12 +17,12 @@
 package com.johnsnowlabs.nlp.annotators.classifier.dl
 
 import com.johnsnowlabs.ml.ai.CamemBertClassification
-import com.johnsnowlabs.ml.ai.util.sentencepiece.{
+import com.johnsnowlabs.ml.tensorflow._
+import com.johnsnowlabs.ml.tensorflow.sentencepiece.{
   ReadSentencePieceModel,
   SentencePieceWrapper,
   WriteSentencePieceModel
 }
-import com.johnsnowlabs.ml.tensorflow._
 import com.johnsnowlabs.ml.util.LoadExternalModel.{
   loadSentencePieceAsset,
   loadTextAsset,

diff --git a/...scala/com/johnsnowlabs/nlp/annotators/classifier/dl/CamemBertForTokenClassification.scala b/...scala/com/johnsnowlabs/nlp/annotators/classifier/dl/CamemBertForTokenClassification.scala
@@ -17,12 +17,12 @@
 package com.johnsnowlabs.nlp.annotators.classifier.dl
 
 import com.johnsnowlabs.ml.ai.CamemBertClassification
-import com.johnsnowlabs.ml.ai.util.sentencepiece.{
+import com.johnsnowlabs.ml.tensorflow._
+import com.johnsnowlabs.ml.tensorflow.sentencepiece.{
   ReadSentencePieceModel,
   SentencePieceWrapper,
   WriteSentencePieceModel
 }
-import com.johnsnowlabs.ml.tensorflow._
 import com.johnsnowlabs.ml.util.LoadExternalModel.{
   loadSentencePieceAsset,
   loadTextAsset,

diff --git a/...ain/scala/com/johnsnowlabs/nlp/annotators/classifier/dl/DeBertaForQuestionAnswering.scala b/...ain/scala/com/johnsnowlabs/nlp/annotators/classifier/dl/DeBertaForQuestionAnswering.scala
@@ -16,13 +16,13 @@
 
 package com.johnsnowlabs.nlp.annotators.classifier.dl
 
-import com.johnsnowlabs.ml.ai.util.sentencepiece.{
+import com.johnsnowlabs.ml.ai.{DeBertaClassification, MergeTokenStrategy}
+import com.johnsnowlabs.ml.tensorflow._
+import com.johnsnowlabs.ml.tensorflow.sentencepiece.{
   ReadSentencePieceModel,
   SentencePieceWrapper,
   WriteSentencePieceModel
 }
-import com.johnsnowlabs.ml.ai.{DeBertaClassification, MergeTokenStrategy}
-import com.johnsnowlabs.ml.tensorflow._
 import com.johnsnowlabs.ml.util.LoadExternalModel.{
   loadSentencePieceAsset,
   modelSanityCheck,

diff --git a/...cala/com/johnsnowlabs/nlp/annotators/classifier/dl/DeBertaForSequenceClassification.scala b/...cala/com/johnsnowlabs/nlp/annotators/classifier/dl/DeBertaForSequenceClassification.scala
@@ -17,12 +17,12 @@
 package com.johnsnowlabs.nlp.annotators.classifier.dl
 
 import com.johnsnowlabs.ml.ai.DeBertaClassification
-import com.johnsnowlabs.ml.ai.util.sentencepiece.{
+import com.johnsnowlabs.ml.tensorflow._
+import com.johnsnowlabs.ml.tensorflow.sentencepiece.{
   ReadSentencePieceModel,
   SentencePieceWrapper,
   WriteSentencePieceModel
 }
-import com.johnsnowlabs.ml.tensorflow._
 import com.johnsnowlabs.ml.util.LoadExternalModel.{
   loadSentencePieceAsset,
   loadTextAsset,

diff --git a/...n/scala/com/johnsnowlabs/nlp/annotators/classifier/dl/DeBertaForTokenClassification.scala b/...n/scala/com/johnsnowlabs/nlp/annotators/classifier/dl/DeBertaForTokenClassification.scala
@@ -17,12 +17,12 @@
 package com.johnsnowlabs.nlp.annotators.classifier.dl
 
 import com.johnsnowlabs.ml.ai.DeBertaClassification
-import com.johnsnowlabs.ml.ai.util.sentencepiece.{
+import com.johnsnowlabs.ml.tensorflow._
+import com.johnsnowlabs.ml.tensorflow.sentencepiece.{
   ReadSentencePieceModel,
   SentencePieceWrapper,
   WriteSentencePieceModel
 }
-import com.johnsnowlabs.ml.tensorflow._
 import com.johnsnowlabs.ml.util.LoadExternalModel.{
   loadSentencePieceAsset,
   loadTextAsset,

diff --git a/.../scala/com/johnsnowlabs/nlp/annotators/classifier/dl/XlmRoBertaForQuestionAnswering.scala b/.../scala/com/johnsnowlabs/nlp/annotators/classifier/dl/XlmRoBertaForQuestionAnswering.scala
@@ -16,13 +16,13 @@
 
 package com.johnsnowlabs.nlp.annotators.classifier.dl
 
-import com.johnsnowlabs.ml.ai.util.sentencepiece.{
+import com.johnsnowlabs.ml.ai.{MergeTokenStrategy, XlmRoBertaClassification}
+import com.johnsnowlabs.ml.tensorflow._
+import com.johnsnowlabs.ml.tensorflow.sentencepiece.{
   ReadSentencePieceModel,
   SentencePieceWrapper,
   WriteSentencePieceModel
 }
-import com.johnsnowlabs.ml.ai.{MergeTokenStrategy, XlmRoBertaClassification}
-import com.johnsnowlabs.ml.tensorflow._
 import com.johnsnowlabs.ml.util.LoadExternalModel.{
   loadSentencePieceAsset,
   modelSanityCheck,

diff --git a/...a/com/johnsnowlabs/nlp/annotators/classifier/dl/XlmRoBertaForSequenceClassification.scala b/...a/com/johnsnowlabs/nlp/annotators/classifier/dl/XlmRoBertaForSequenceClassification.scala
@@ -17,12 +17,12 @@
 package com.johnsnowlabs.nlp.annotators.classifier.dl
 
 import com.johnsnowlabs.ml.ai.XlmRoBertaClassification
-import com.johnsnowlabs.ml.ai.util.sentencepiece.{
+import com.johnsnowlabs.ml.tensorflow._
+import com.johnsnowlabs.ml.tensorflow.sentencepiece.{
   ReadSentencePieceModel,
   SentencePieceWrapper,
   WriteSentencePieceModel
 }
-import com.johnsnowlabs.ml.tensorflow._
 import com.johnsnowlabs.ml.util.LoadExternalModel.{
   loadSentencePieceAsset,
   loadTextAsset,

diff --git a/...cala/com/johnsnowlabs/nlp/annotators/classifier/dl/XlmRoBertaForTokenClassification.scala b/...cala/com/johnsnowlabs/nlp/annotators/classifier/dl/XlmRoBertaForTokenClassification.scala
@@ -17,12 +17,12 @@
 package com.johnsnowlabs.nlp.annotators.classifier.dl
 
 import com.johnsnowlabs.ml.ai.XlmRoBertaClassification
-import com.johnsnowlabs.ml.ai.util.sentencepiece.{
+import com.johnsnowlabs.ml.tensorflow._
+import com.johnsnowlabs.ml.tensorflow.sentencepiece.{
   ReadSentencePieceModel,
   SentencePieceWrapper,
   WriteSentencePieceModel
 }
-import com.johnsnowlabs.ml.tensorflow._
 import com.johnsnowlabs.ml.util.LoadExternalModel.{
   loadSentencePieceAsset,
   loadTextAsset,

diff --git a/.../scala/com/johnsnowlabs/nlp/annotators/classifier/dl/XlnetForSequenceClassification.scala b/.../scala/com/johnsnowlabs/nlp/annotators/classifier/dl/XlnetForSequenceClassification.scala
@@ -17,12 +17,12 @@
 package com.johnsnowlabs.nlp.annotators.classifier.dl
 
 import com.johnsnowlabs.ml.ai.XlnetClassification
-import com.johnsnowlabs.ml.ai.util.sentencepiece.{
+import com.johnsnowlabs.ml.tensorflow._
+import com.johnsnowlabs.ml.tensorflow.sentencepiece.{
   ReadSentencePieceModel,
   SentencePieceWrapper,
   WriteSentencePieceModel
 }
-import com.johnsnowlabs.ml.tensorflow._
 import com.johnsnowlabs.ml.util.LoadExternalModel.{
   loadSentencePieceAsset,
   loadTextAsset,

diff --git a/...ain/scala/com/johnsnowlabs/nlp/annotators/classifier/dl/XlnetForTokenClassification.scala b/...ain/scala/com/johnsnowlabs/nlp/annotators/classifier/dl/XlnetForTokenClassification.scala
@@ -17,12 +17,12 @@
 package com.johnsnowlabs.nlp.annotators.classifier.dl
 
 import com.johnsnowlabs.ml.ai.XlnetClassification
-import com.johnsnowlabs.ml.ai.util.sentencepiece.{
+import com.johnsnowlabs.ml.tensorflow._
+import com.johnsnowlabs.ml.tensorflow.sentencepiece.{
   ReadSentencePieceModel,
   SentencePieceWrapper,
   WriteSentencePieceModel
 }
-import com.johnsnowlabs.ml.tensorflow._
 import com.johnsnowlabs.ml.util.LoadExternalModel.{
   loadSentencePieceAsset,
   loadTextAsset,

diff --git a/src/main/scala/com/johnsnowlabs/nlp/annotators/seq2seq/T5Transformer.scala b/src/main/scala/com/johnsnowlabs/nlp/annotators/seq2seq/T5Transformer.scala
@@ -17,7 +17,7 @@
 package com.johnsnowlabs.nlp.annotators.seq2seq
 
 import com.johnsnowlabs.ml.ai.T5
-import com.johnsnowlabs.ml.ai.util.sentencepiece.{
+import com.johnsnowlabs.ml.tensorflow.sentencepiece.{
   ReadSentencePieceModel,
   SentencePieceWrapper,
   WriteSentencePieceModel