-
Notifications
You must be signed in to change notification settings - Fork 718
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #78 from JohnSnowLabs/serialization_features_bc
Serialization features bc
- Loading branch information
Showing
14 changed files
with
354 additions
and
181 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| package com.johnsnowlabs.nlp | ||
|
|
||
| import com.johnsnowlabs.nlp.serialization.{ArrayFeature, Feature, MapFeature, StructFeature} | ||
|
|
||
| import scala.collection.mutable.ArrayBuffer | ||
|
|
||
| trait HasFeatures { | ||
|
|
||
| val features: ArrayBuffer[Feature[_, _, _]] = ArrayBuffer.empty | ||
|
|
||
| protected def set[T](feature: ArrayFeature[T], value: Array[T]): this.type = {feature.setValue(Some(value)); this} | ||
|
|
||
| protected def set[K, V](feature: MapFeature[K, V], value: Map[K, V]): this.type = {feature.setValue(Some(value)); this} | ||
|
|
||
| protected def set[T](feature: StructFeature[T], value: T): this.type = {feature.setValue(Some(value)); this} | ||
|
|
||
| protected def setDefault[T](feature: ArrayFeature[T], value: Array[T]): this.type = {feature.setValue(Some(value)); this} | ||
|
|
||
| protected def setDefault[K, V](feature: MapFeature[K, V], value: Map[K, V]): this.type = {feature.setValue(Some(value)); this} | ||
|
|
||
| protected def setDefault[T](feature: StructFeature[T], value: T): this.type = {feature.setValue(Some(value)); this} | ||
|
|
||
| protected def get[T](feature: ArrayFeature[T]): Option[Array[T]] = feature.get | ||
|
|
||
| protected def get[K, V](feature: MapFeature[K, V]): Option[Map[K, V]] = feature.get | ||
|
|
||
| protected def get[T](feature: StructFeature[T]): Option[T] = feature.get | ||
|
|
||
| protected def $$[T](feature: ArrayFeature[T]): Array[T] = feature.getValue | ||
|
|
||
| protected def $$[K, V](feature: MapFeature[K, V]): Map[K, V] = feature.getValue | ||
|
|
||
| protected def $$[T](feature: StructFeature[T]): T = feature.getValue | ||
|
|
||
| } |
31 changes: 31 additions & 0 deletions
31
src/main/scala/com/johnsnowlabs/nlp/ParamsAndFeaturesReadable.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| package com.johnsnowlabs.nlp | ||
|
|
||
| import org.apache.spark.ml.util.{DefaultParamsReadable, MLReader} | ||
| import org.apache.spark.sql.SparkSession | ||
|
|
||
| class FeaturesReader[T <: HasFeatures](baseReader: MLReader[T], onRead: (T, String, SparkSession) => Unit) extends MLReader[T] { | ||
|
|
||
| override def load(path: String): T = { | ||
|
|
||
| val instance = baseReader.load(path) | ||
|
|
||
| for (feature <- instance.features) { | ||
| val value = feature.deserialize(sparkSession, path, feature.name) | ||
| feature.setValue(value) | ||
| } | ||
|
|
||
| onRead(instance, path, sparkSession) | ||
|
|
||
| instance | ||
| } | ||
| } | ||
|
|
||
| trait ParamsAndFeaturesReadable[T <: HasFeatures] extends DefaultParamsReadable[T] { | ||
|
|
||
| def onRead(instance: T, path: String, spark: SparkSession): Unit = {} | ||
|
|
||
| override def read: MLReader[T] = new FeaturesReader( | ||
| super.read, | ||
| (instance: T, path: String, spark: SparkSession) => onRead(instance, path, spark) | ||
| ) | ||
| } |
32 changes: 32 additions & 0 deletions
32
src/main/scala/com/johnsnowlabs/nlp/ParamsAndFeaturesWritable.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,32 @@ | ||
| package com.johnsnowlabs.nlp | ||
|
|
||
| import org.apache.spark.ml.param.Params | ||
| import org.apache.spark.ml.util.{DefaultParamsWritable, MLWriter} | ||
| import org.apache.spark.sql.SparkSession | ||
|
|
||
| class FeaturesWriter[T](annotatorWithFeatures: HasFeatures, baseWriter: MLWriter, onWritten: (String, SparkSession) => Unit) | ||
| extends MLWriter with HasFeatures { | ||
|
|
||
| override protected def saveImpl(path: String): Unit = { | ||
| baseWriter.save(path) | ||
|
|
||
| for (feature <- annotatorWithFeatures.features) { | ||
| feature.serializeInfer(sparkSession, path, feature.name, feature.getValue) | ||
| } | ||
|
|
||
| onWritten(path, sparkSession) | ||
|
|
||
| } | ||
| } | ||
|
|
||
| trait ParamsAndFeaturesWritable extends DefaultParamsWritable with Params with HasFeatures { | ||
|
|
||
| def onWritten(path: String, spark: SparkSession): Unit = {} | ||
|
|
||
| override def write: MLWriter = new FeaturesWriter( | ||
| this, | ||
| super.write, | ||
| (path: String, spark: SparkSession) => onWritten(path, spark) | ||
| ) | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.