adding openvino support to all ClassificationForXXX annotators (#14408)

* adding DebertaForXXX support * Uploading the remaining files * Delete examples/python/transformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_CamemBERT.ipynbshield * Delete examples/python/transformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_XlmRoBertaForTokenClassification.ipynbshield * Delete examples/python/transformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_XlmRoBertaForZeroShotClassification.ipynbshield * adding openvino Suppor tto multiple Annotators * Update Bart.scala
JohnSnowLabs · Dec 9, 2024 · 4b2aa30 · 4b2aa30
1 parent 6653546
commit 4b2aa30
Show file tree

Hide file tree

Showing 124 changed files with 89,872 additions and 1,839 deletions.
diff --git a/examples/python/transformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_ALBERT.ipynb b/examples/python/transformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_ALBERT.ipynb
diff --git a/examples/python/transformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_BGE.ipynb b/examples/python/transformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_BGE.ipynb
diff --git a/examples/python/transformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_CLIP.ipynb b/examples/python/transformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_CLIP.ipynb
diff --git a/examples/python/transformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_CamemBERT.ipynb b/examples/python/transformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_CamemBERT.ipynb
diff --git a/...nsformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_ConvNextForImageClassification.ipynb b/...nsformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_ConvNextForImageClassification.ipynb
diff --git a/examples/python/transformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_DeBERTa.ipynb b/examples/python/transformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_DeBERTa.ipynb
diff --git a/...transformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_DeBertaForQuestionAnswering.ipynb b/...transformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_DeBertaForQuestionAnswering.ipynb
diff --git a/...formers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_DeBertaForSequenceClassification.ipynb b/...formers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_DeBertaForSequenceClassification.ipynb
diff --git a/...ansformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_DeBertaForTokenClassification.ipynb b/...ansformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_DeBertaForTokenClassification.ipynb
diff --git a/...formers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_DeBertaForZeroShotClassification.ipynb b/...formers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_DeBertaForZeroShotClassification.ipynb
diff --git a/examples/python/transformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_DistilBERT.ipynb b/examples/python/transformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_DistilBERT.ipynb
diff --git a/...nsformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_DistilBertForQuestionAnswering.ipynb b/...nsformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_DistilBertForQuestionAnswering.ipynb
diff --git a/...formers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_DistilBertForTokenClassification.ipynb b/...formers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_DistilBertForTokenClassification.ipynb
diff --git a/...mers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_DistilBertForZeroShotClassification.ipynb b/...mers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_DistilBertForZeroShotClassification.ipynb
diff --git a/...rmers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_DistlBertForSequenceClassification.ipynb b/...rmers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_DistlBertForSequenceClassification.ipynb
diff --git a/examples/python/transformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_GPT2.ipynb b/examples/python/transformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_GPT2.ipynb
diff --git a/examples/python/transformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_Hubert.ipynb b/examples/python/transformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_Hubert.ipynb
diff --git a/examples/python/transformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_Instructor.ipynb b/examples/python/transformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_Instructor.ipynb
diff --git a/...n/transformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_MPNetForQuestionAnswering.ipynb b/...n/transformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_MPNetForQuestionAnswering.ipynb
diff --git a/...transformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_MPNetForTokenClassification.ipynb b/...transformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_MPNetForTokenClassification.ipynb
diff --git a/examples/python/transformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_MPNet_.ipynb b/examples/python/transformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_MPNet_.ipynb
diff --git a/...transformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_RoBertaForQuestionAnswering.ipynb b/...transformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_RoBertaForQuestionAnswering.ipynb
diff --git a/...formers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_RoBertaForSequenceClassification.ipynb b/...formers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_RoBertaForSequenceClassification.ipynb
diff --git a/...ansformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_RoBertaForTokenClassification.ipynb b/...ansformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_RoBertaForTokenClassification.ipynb
diff --git a/.../transformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_SwinForImageClassification.ipynb b/.../transformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_SwinForImageClassification.ipynb
diff --git a/examples/python/transformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_UAE.ipynb b/examples/python/transformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_UAE.ipynb
diff --git a/.../transformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_ViTForImageClassification_.ipynb b/.../transformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_ViTForImageClassification_.ipynb
diff --git a/...s/openvino/HuggingFace_OpenVINO_in_Spark_NLP_VisionEncoderDecoderForImageCaptioning.ipynb b/...s/openvino/HuggingFace_OpenVINO_in_Spark_NLP_VisionEncoderDecoderForImageCaptioning.ipynb
diff --git a/examples/python/transformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_Whisper.ipynb b/examples/python/transformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_Whisper.ipynb
diff --git a/...nsformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_XlmRoBertaForQuestionAnswering.ipynb b/...nsformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_XlmRoBertaForQuestionAnswering.ipynb
diff --git a/...mers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_XlmRoBertaForSequenceClassification.ipynb b/...mers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_XlmRoBertaForSequenceClassification.ipynb
diff --git a/...formers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_XlmRoBertaForTokenClassification.ipynb b/...formers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_XlmRoBertaForTokenClassification.ipynb
diff --git a/...mers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_XlmRoBertaForZeroShotClassification.ipynb b/...mers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_XlmRoBertaForZeroShotClassification.ipynb
diff --git a/...ransformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_XlmRoBertaSentenceEmbeddings.ipynb b/...ransformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_XlmRoBertaSentenceEmbeddings.ipynb
diff --git a/examples/python/transformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_snowflake_.ipynb b/examples/python/transformers/openvino/HuggingFace_OpenVINO_in_Spark_NLP_snowflake_.ipynb
diff --git a/...transformers/openvino/HuggingFace_OpenVino_Spark_NLP_MPNetForSequenceClassification.ipynb b/...transformers/openvino/HuggingFace_OpenVino_Spark_NLP_MPNetForSequenceClassification.ipynb
diff --git a/src/main/scala/com/johnsnowlabs/ml/ai/Albert.scala b/src/main/scala/com/johnsnowlabs/ml/ai/Albert.scala
@@ -19,11 +19,13 @@ package com.johnsnowlabs.ml.ai
 import ai.onnxruntime.OnnxTensor
 import com.johnsnowlabs.ml.ai.util.PrepareEmbeddings
 import com.johnsnowlabs.ml.onnx.{OnnxSession, OnnxWrapper}
+import com.johnsnowlabs.ml.openvino.OpenvinoWrapper
 import com.johnsnowlabs.ml.tensorflow.sentencepiece.{SentencePieceWrapper, SentencepieceEncoder}
 import com.johnsnowlabs.ml.tensorflow.sign.{ModelSignatureConstants, ModelSignatureManager}
 import com.johnsnowlabs.ml.tensorflow.{TensorResources, TensorflowWrapper}
-import com.johnsnowlabs.ml.util.{ONNX, TensorFlow}
+import com.johnsnowlabs.ml.util.{ONNX, Openvino, TensorFlow}
 import com.johnsnowlabs.nlp.annotators.common._
+import org.intel.openvino.Tensor
 import org.slf4j.{Logger, LoggerFactory}
 
 import scala.collection.JavaConverters._
@@ -71,6 +73,7 @@ import scala.collection.JavaConverters._
 private[johnsnowlabs] class Albert(
     val tensorflowWrapper: Option[TensorflowWrapper],
     val onnxWrapper: Option[OnnxWrapper],
+    val openvinoWrapper: Option[OpenvinoWrapper],
     val spp: SentencePieceWrapper,
     batchSize: Int,
     configProtoBytes: Option[Array[Byte]] = None,
@@ -83,6 +86,7 @@ private[johnsnowlabs] class Albert(
   val detectedEngine: String =
     if (tensorflowWrapper.isDefined) TensorFlow.name
     else if (onnxWrapper.isDefined) ONNX.name
+    else if (openvinoWrapper.isDefined) Openvino.name
     else TensorFlow.name
   private val onnxSessionOptions: Map[String, String] = new OnnxSession().getSessionOptions
 
@@ -155,6 +159,41 @@ private[johnsnowlabs] class Albert(
           maskTensors.close()
           segmentTensors.close()
         }
+
+
+      case Openvino.name =>
+
+
+
+        val batchLength = batch.length
+        val shape = Array(batchLength, maxSentenceLength)
+        val (tokenTensors, maskTensors) =
+          PrepareEmbeddings.prepareOvLongBatchTensors(batch, maxSentenceLength, batchLength)
+        val segmentTensors = new Tensor(shape, Array.fill(batchLength * maxSentenceLength)(0L))
+
+        val inferRequest = openvinoWrapper.get.getCompiledModel().create_infer_request()
+        inferRequest.set_tensor("input_ids", tokenTensors)
+        inferRequest.set_tensor("attention_mask", maskTensors)
+        inferRequest.set_tensor("token_type_ids", segmentTensors)
+
+        inferRequest.infer()
+
+        try {
+          try {
+            inferRequest
+              .get_tensor("last_hidden_state")
+              .data()
+          }
+        } catch {
+          case e: Exception =>
+            e.printStackTrace()
+            Array.empty[Float]
+            // Rethrow the exception to propagate it further
+            throw e
+        }
+
+
+
       case _ =>
         val tensors = new TensorResources()
 

diff --git a/src/main/scala/com/johnsnowlabs/ml/ai/BGE.scala b/src/main/scala/com/johnsnowlabs/ml/ai/BGE.scala
@@ -17,12 +17,15 @@
 package com.johnsnowlabs.ml.ai
 
 import ai.onnxruntime.{OnnxTensor, TensorInfo}
+import com.johnsnowlabs.ml.ai.util.PrepareEmbeddings
 import com.johnsnowlabs.ml.onnx.{OnnxSession, OnnxWrapper}
+import com.johnsnowlabs.ml.openvino.OpenvinoWrapper
 import com.johnsnowlabs.ml.tensorflow.sign.{ModelSignatureConstants, ModelSignatureManager}
 import com.johnsnowlabs.ml.tensorflow.{TensorResources, TensorflowWrapper}
-import com.johnsnowlabs.ml.util.{LinAlg, ONNX, TensorFlow}
+import com.johnsnowlabs.ml.util.{LinAlg, ONNX, Openvino, TensorFlow}
 import com.johnsnowlabs.nlp.annotators.common._
 import com.johnsnowlabs.nlp.{Annotation, AnnotatorType}
+import org.intel.openvino.Tensor
 import org.slf4j.{Logger, LoggerFactory}
 
 import scala.collection.JavaConverters._
@@ -42,6 +45,7 @@ import scala.collection.JavaConverters._
 private[johnsnowlabs] class BGE(
     val tensorflowWrapper: Option[TensorflowWrapper],
     val onnxWrapper: Option[OnnxWrapper],
+    val openvinoWrapper: Option[OpenvinoWrapper],
     configProtoBytes: Option[Array[Byte]] = None,
     sentenceStartTokenId: Int,
     sentenceEndTokenId: Int,
@@ -57,6 +61,7 @@ private[johnsnowlabs] class BGE(
   val detectedEngine: String =
     if (tensorflowWrapper.isDefined) TensorFlow.name
     else if (onnxWrapper.isDefined) ONNX.name
+    else if (openvinoWrapper.isDefined) Openvino.name
     else TensorFlow.name
   private val onnxSessionOptions: Map[String, String] = new OnnxSession().getSessionOptions
 
@@ -72,6 +77,9 @@ private[johnsnowlabs] class BGE(
     val embeddings = detectedEngine match {
       case ONNX.name =>
         getSentenceEmbeddingFromOnnx(paddedBatch, maxSentenceLength)
+
+      case Openvino.name =>
+        getSentenceEmbeddingFromOv(paddedBatch, maxSentenceLength)
       case _ =>
         getSentenceEmbeddingFromTF(paddedBatch, maxSentenceLength)
     }
@@ -160,6 +168,54 @@ private[johnsnowlabs] class BGE(
     sentenceEmbeddingsFloatsArray
   }
 
+
+
+  private def getSentenceEmbeddingFromOv(
+                                            batch: Seq[Array[Int]],
+                                            maxSentenceLength: Int): Array[Array[Float]] = {
+
+
+    val batchLength = batch.length
+    val shape = Array(batchLength, maxSentenceLength)
+    val tokenTensors =
+      new org.intel.openvino.Tensor(shape, batch.flatMap(x => x.map(xx => xx.toLong)).toArray)
+    val attentionMask = batch.map(sentence => sentence.map(x => if (x < 0L) 0L else 1L)).toArray
+
+    val maskTensors = new org.intel.openvino.Tensor(
+      shape,
+      attentionMask.flatten)
+
+    val segmentTensors = new Tensor(shape, Array.fill(batchLength * maxSentenceLength)(0L))
+    val inferRequest = openvinoWrapper.get.getCompiledModel().create_infer_request()
+    inferRequest.set_tensor("input_ids", tokenTensors)
+    inferRequest.set_tensor("attention_mask", maskTensors)
+    inferRequest.set_tensor("token_type_ids", segmentTensors)
+
+    inferRequest.infer()
+
+    try {
+      try {
+        val lastHiddenState = inferRequest
+          .get_tensor("last_hidden_state")
+        val shape = lastHiddenState.get_shape().map(_.toLong)
+       val flattenEmbeddings =  lastHiddenState
+          .data()
+        val embeddings = LinAlg.avgPooling(flattenEmbeddings, attentionMask, shape)
+        val normalizedEmbeddings = LinAlg.l2Normalize(embeddings)
+        LinAlg.denseMatrixToArray(normalizedEmbeddings)
+
+      }
+    } catch {
+      case e: Exception =>
+        e.printStackTrace()
+        Array.empty[Float]
+        // Rethrow the exception to propagate it further
+        throw e
+    }
+
+  }
+
+
   private def getSentenceEmbeddingFromOnnx(
       batch: Seq[Array[Int]],
       maxSentenceLength: Int): Array[Array[Float]] = {