Merge pull request #531 from JohnSnowLabs/207-release-candidate

Release version bump

CHANGELOG

@@ -1,3 +1,21 @@
+========
+2.0.7
+========
+---------------
+Overview
+---------------
+This release addresses bugs related to cluster support, improving error messages and fixing various potential bugs depending
+on the cluster configuration, such as Kryo Serialization or non default FS systems
+
+---------------
+Bugfixes
+---------------
+* Fixed a bug introduced in 2.0.5 that caused NerDL not to work in clusters with Kryo serialization enabled
+* NerDLModel was not properly reading user provided config proto bytes during prediction
+* Improved cluster embeddings message to hit user of cluster mode without shared filesystems
+* Removed lazy model downloading on PretrainedPipeline to download the model at instantiation
+* Fixed URI construction for cluster embeddings on non defaultFS configurations, improves cluster compatibility
+
 ========
 2.0.6
 ========

README.md

@@ -40,7 +40,7 @@ Take a look at our official Spark NLP page: [http://nlp.johnsnowlabs.com/](http:
 
 ## Apache Spark Support
 
-Spark NLP *2.0.6* has been built on top of Apache Spark 2.4.3
+Spark NLP *2.0.7* has been built on top of Apache Spark 2.4.3
 
 Note that Spark is not retrocompatible with Spark 2.3.x, so models and environments might not work.
 
@@ -65,18 +65,18 @@ This library has been uploaded to the [spark-packages repository](https://spark-
 
 Benefit of spark-packages is that makes it available for both Scala-Java and Python
 
-To use the most recent version just add the `--packages JohnSnowLabs:spark-nlp:2.0.6` to you spark command
+To use the most recent version just add the `--packages JohnSnowLabs:spark-nlp:2.0.7` to you spark command
 
 ```sh
-spark-shell --packages JohnSnowLabs:spark-nlp:2.0.6
+spark-shell --packages JohnSnowLabs:spark-nlp:2.0.7
 ```
 
 ```sh
-pyspark --packages JohnSnowLabs:spark-nlp:2.0.6
+pyspark --packages JohnSnowLabs:spark-nlp:2.0.7
 ```
 
 ```sh
-spark-submit --packages JohnSnowLabs:spark-nlp:2.0.6
+spark-submit --packages JohnSnowLabs:spark-nlp:2.0.7
 ```
 
 This can also be used to create a SparkSession manually by using the `spark.jars.packages` option in both Python and Scala
@@ -144,7 +144,7 @@ Our package is deployed to maven central. In order to add this package as a depe
 <dependency>
     <groupId>com.johnsnowlabs.nlp</groupId>
     <artifactId>spark-nlp_2.11</artifactId>
-    <version>2.0.6</version>
+    <version>2.0.7</version>
 </dependency>
 ```
 
@@ -155,22 +155,22 @@ and
 <dependency>
     <groupId>com.johnsnowlabs.nlp</groupId>
     <artifactId>spark-nlp-ocr_2.11</artifactId>
-    <version>2.0.6</version>
+    <version>2.0.7</version>
 </dependency>
 ```
 
 ### SBT
 
 ```sbtshell
 // https://mvnrepository.com/artifact/com.johnsnowlabs.nlp/spark-nlp
-libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp" % "2.0.6"
+libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp" % "2.0.7"
 ```
 
 and
 
 ```sbtshell
 // https://mvnrepository.com/artifact/com.johnsnowlabs.nlp/spark-nlp-ocr
-libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp-ocr" % "2.0.6"
+libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp-ocr" % "2.0.7"
 ```
 
 Maven Central: [https://mvnrepository.com/artifact/com.johnsnowlabs.nlp](https://mvnrepository.com/artifact/com.johnsnowlabs.nlp)
@@ -184,7 +184,7 @@ Maven Central: [https://mvnrepository.com/artifact/com.johnsnowlabs.nlp](https:/
 If you installed pyspark through pip, you can install `spark-nlp` through pip as well.
 
 ```bash
-pip install spark-nlp==2.0.6
+pip install spark-nlp==2.0.7
 ```
 
 PyPI [spark-nlp package](https://pypi.org/project/spark-nlp/)
@@ -207,7 +207,7 @@ spark = SparkSession.builder \
     .master("local[4]")\
     .config("spark.driver.memory","4G")\
     .config("spark.driver.maxResultSize", "2G") \
-    .config("spark.jars.packages", "JohnSnowLabs:spark-nlp:2.0.6")\
+    .config("spark.jars.packages", "JohnSnowLabs:spark-nlp:2.0.7")\
     .config("spark.kryoserializer.buffer.max", "500m")\
     .getOrCreate()
 ```
@@ -221,7 +221,7 @@ Use either one of the following options
 * Add the following Maven Coordinates to the interpreter's library list
 
 ```bash
-com.johnsnowlabs.nlp:spark-nlp_2.11:2.0.6
+com.johnsnowlabs.nlp:spark-nlp_2.11:2.0.7
 ```
 
 * Add path to pre-built jar from [here](#pre-compiled-spark-nlp-and-spark-nlp-ocr) in the interpreter's library list making sure the jar is available to driver path
@@ -231,7 +231,7 @@ com.johnsnowlabs.nlp:spark-nlp_2.11:2.0.6
 Apart from previous step, install python module through pip
 
 ```bash
-pip install spark-nlp==2.0.6
+pip install spark-nlp==2.0.7
 ```
 
 Or you can install `spark-nlp` from inside Zeppelin by using Conda:
@@ -256,7 +256,7 @@ export PYSPARK_PYTHON=python3
 export PYSPARK_DRIVER_PYTHON=jupyter
 export PYSPARK_DRIVER_PYTHON_OPTS=notebook
 
-pyspark --packages JohnSnowLabs:spark-nlp:2.0.6
+pyspark --packages JohnSnowLabs:spark-nlp:2.0.7
 ```
 
 Alternatively, you can mix in using `--jars` option for pyspark + `pip install spark-nlp`

build.sbt

@@ -16,7 +16,7 @@ if(is_gpu.equals("false")){
 
 organization:= "com.johnsnowlabs.nlp"
 
-version := "2.0.6"
+version := "2.0.7"
 
 scalaVersion in ThisBuild := scalaVer
 
@@ -178,7 +178,7 @@ assemblyMergeStrategy in assembly := {
 lazy val ocr = (project in file("ocr"))
   .settings(
     name := "spark-nlp-ocr",
-    version := "2.0.6",
+    version := "2.0.7",
 
     test in assembly := {},
 

docs/Gemfile.lock

@@ -211,7 +211,7 @@ GEM
       sawyer (~> 0.8.0, >= 0.5.3)
     pathutil (0.16.2)
       forwardable-extended (~> 2.6)
-    public_suffix (2.0.6)
+    public_suffix (2.0.7)
     rb-fsevent (0.10.3)
     rb-inotify (0.10.0)
       ffi (~> 1.0)

docs/_layouts/landing.html

@@ -49,22 +49,22 @@ <h1>{{ _section.title }}</h1>
           <div class="cell cell--12 cell--lg-12" style="text-align: left; background-color: #2d2d2d; padding: 10px">
             {% highlight bash %}
 # Install Spark NLP from PyPI
-$ pip install spark-nlp==2.0.6
+$ pip install spark-nlp==2.0.7
 
 # Install Spark NLP from Anacodna/Conda
 $ conda install -c johnsnowlabs spark-nlp
 
 # Load Spark NLP with Spark Shell
-$ spark-shell --packages JohnSnowLabs:spark-nlp:2.0.6
+$ spark-shell --packages JohnSnowLabs:spark-nlp:2.0.7
 
 # Load Spark NLP with PySpark
-$ pyspark --packages JohnSnowLabs:spark-nlp:2.0.6
+$ pyspark --packages JohnSnowLabs:spark-nlp:2.0.7
 
 # Load Spark NLP with Spark Submit
-$ spark-submit --packages JohnSnowLabs:spark-nlp:2.0.6
+$ spark-submit --packages JohnSnowLabs:spark-nlp:2.0.7
 
 # Load Spark NLP as external JAR after comiling and bulding Spark NLP by `sbt assembly`
-$ spark-shell --jar spark-nlp-assembly-2.0.6
+$ spark-shell --jar spark-nlp-assembly-2.0.7
             {% endhighlight %}
           </div>
         </div>

docs/en/install.md

@@ -13,7 +13,7 @@ modify_date: "2019-05-16"
 If you installed pyspark through pip, you can install `spark-nlp` through pip as well.
 
 ```bash
-pip install spark-nlp==2.0.6
+pip install spark-nlp==2.0.7
 ```
 
 PyPI [spark-nlp package](https://pypi.org/project/spark-nlp/)
@@ -36,7 +36,7 @@ spark = SparkSession.builder \
     .master("local[*]")\
     .config("spark.driver.memory","8G")\
     .config("spark.driver.maxResultSize", "2G") \
-    .config("spark.jars.packages", "JohnSnowLabs:spark-nlp:2.0.6")\
+    .config("spark.jars.packages", "JohnSnowLabs:spark-nlp:2.0.7")\
     .config("spark.kryoserializer.buffer.max", "500m")\
     .getOrCreate()
 ```
@@ -97,7 +97,7 @@ Our package is deployed to maven central. In order to add this package as a depe
 <dependency>
     <groupId>com.johnsnowlabs.nlp</groupId>
     <artifactId>spark-nlp_2.11</artifactId>
-    <version>2.0.6</version>
+    <version>2.0.7</version>
 </dependency>
 ```
 
@@ -108,22 +108,22 @@ and
 <dependency>
     <groupId>com.johnsnowlabs.nlp</groupId>
     <artifactId>spark-nlp-ocr_2.11</artifactId>
-    <version>2.0.6</version>
+    <version>2.0.7</version>
 </dependency>
 ```
 
 ### SBT
 
 ```sbtshell
 // https://mvnrepository.com/artifact/com.johnsnowlabs.nlp/spark-nlp
-libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp" % "2.0.6"
+libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp" % "2.0.7"
 ```
 
 and
 
 ```sbtshell
 // https://mvnrepository.com/artifact/com.johnsnowlabs.nlp/spark-nlp-ocr
-libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp-ocr" % "2.0.6"
+libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp-ocr" % "2.0.7"
 ```
 
 Maven Central: [https://mvnrepository.com/artifact/com.johnsnowlabs.nlp](https://mvnrepository.com/artifact/com.johnsnowlabs.nlp)
@@ -151,7 +151,7 @@ Note: You can import these notebooks by using their URLs.
 4- From the Source drop-down menu, select **Maven Coordinate:**
 ![Databricks](https://databricks.com/wp-content/uploads/2015/07/select-maven-1024x711.png)
 
-5- Now, all available **Spark Packages** are at your fingertips! Just search for **JohnSnowLabs:spark-nlp:version** where **version** stands for the library version such as: `1.8.4` or `2.0.6`
+5- Now, all available **Spark Packages** are at your fingertips! Just search for **JohnSnowLabs:spark-nlp:version** where **version** stands for the library version such as: `1.8.4` or `2.0.7`
 ![Databricks](https://databricks.com/wp-content/uploads/2015/07/browser-1024x548.png)
 
 6- Select **spark-nlp** package and we are good to go!

docs/en/quickstart.md

@@ -29,17 +29,17 @@ Spark NLP is built on top of **Apache Spark 2.4.0** and such is the **only** sup
 To start using the library, execute any of the following lines depending on your desired use case:
 
 ```bash
-spark-shell --packages JohnSnowLabs:spark-nlp:2.0.6
-pyspark --packages JohnSnowLabs:spark-nlp:2.0.6
-spark-submit --packages JohnSnowLabs:spark-nlp:2.0.6
+spark-shell --packages JohnSnowLabs:spark-nlp:2.0.7
+pyspark --packages JohnSnowLabs:spark-nlp:2.0.7
+spark-submit --packages JohnSnowLabs:spark-nlp:2.0.7
 ```
 
 ### **Straight forward Python on jupyter notebook**
 
 Use pip to install (after you pip installed numpy and pyspark)
 
 ```bash
-pip install spark-nlp==2.0.6
+pip install spark-nlp==2.0.7
 jupyter notebook
 ```
 
@@ -60,7 +60,7 @@ spark = SparkSession.builder \
     .appName('OCR Eval') \
     .config("spark.driver.memory", "6g") \
     .config("spark.executor.memory", "6g") \
-    .config("spark.jars.packages", "JohnSnowLabs:spark-nlp:2.0.6") \
+    .config("spark.jars.packages", "JohnSnowLabs:spark-nlp:2.0.7") \
     .getOrCreate()
 ```
 
@@ -69,13 +69,13 @@ spark = SparkSession.builder \
 Add the following maven coordinates in the dependency configuration page:
 
 ```bash
-com.johnsnowlabs.nlp:spark-nlp_2.11:2.0.6
+com.johnsnowlabs.nlp:spark-nlp_2.11:2.0.7
 ```
 
 For Python in **Apache Zeppelin** you may need to setup _**SPARK_SUBMIT_OPTIONS**_ utilizing --packages instruction shown above like this
 
 ```bash
-export SPARK_SUBMIT_OPTIONS="--packages JohnSnowLabs:spark-nlp:2.0.6"
+export SPARK_SUBMIT_OPTIONS="--packages JohnSnowLabs:spark-nlp:2.0.7"
 ```
 
 ### **Python Jupyter Notebook with PySpark**
@@ -85,7 +85,7 @@ export SPARK_HOME=/path/to/your/spark/folder
 export PYSPARK_DRIVER_PYTHON=jupyter
 export PYSPARK_DRIVER_PYTHON_OPTS=notebook
 
-pyspark --packages JohnSnowLabs:spark-nlp:2.0.6
+pyspark --packages JohnSnowLabs:spark-nlp:2.0.7
 ```
 
 ### S3 based standalone cluster (No Hadoop)
@@ -297,7 +297,7 @@ lightPipeline.annotate("Hello world, please annotate my text")
 Spark NLP OCR Module is not included within Spark NLP. It is not an annotator and not an extension to Spark ML. You can include it with the following coordinates for Maven:
 
 ```bash
-com.johnsnowlabs.nlp:spark-nlp-ocr_2.11:2.0.6
+com.johnsnowlabs.nlp:spark-nlp-ocr_2.11:2.0.7
 ```
 
 ### Creating Spark datasets from PDF (To be used with Spark NLP)

python/setup.py

@@ -40,7 +40,7 @@
     # For a discussion on single-sourcing the version across setup.py and the
     # project code, see
     # https://packaging.python.org/en/latest/single_source_version.html
-    version='2.0.6',  # Required
+    version='2.0.7',  # Required
 
     # This is a one-line description or tagline of what your project does. This
     # corresponds to the "Summary" metadata field:

python/sparknlp/__init__.py

@@ -40,14 +40,14 @@ def start(include_ocr=False):
 
     if include_ocr:
         builder \
-            .config("spark.jars.packages", "JohnSnowLabs:spark-nlp:2.0.6,com.johnsnowlabs.nlp:spark-nlp-ocr_2.11:2.0.6,javax.media.jai:com.springsource.javax.media.jai.core:1.1.3") \
+            .config("spark.jars.packages", "JohnSnowLabs:spark-nlp:2.0.7,com.johnsnowlabs.nlp:spark-nlp-ocr_2.11:2.0.7,javax.media.jai:com.springsource.javax.media.jai.core:1.1.3") \
             .config("spark.jars.repositories", "http://repo.spring.io/plugins-release")
 
     else:
-        builder.config("spark.jars.packages", "JohnSnowLabs:spark-nlp:2.0.6") \
+        builder.config("spark.jars.packages", "JohnSnowLabs:spark-nlp:2.0.7") \
 
     return builder.getOrCreate()
 
 
 def version():
-    print('2.0.6')
+    print('2.0.7')

src/main/scala/com/johnsnowlabs/nlp/SparkNLP.scala

@@ -13,18 +13,18 @@ object SparkNLP {
 
     if (includeOcr) {
       build
-        .config("spark.jars.packages", "JohnSnowLabs:spark-nlp:2.0.6,com.johnsnowlabs.nlp:spark-nlp-ocr_2.11:2.0.6,javax.media.jai:com.springsource.javax.media.jai.core:1.1.3")
+        .config("spark.jars.packages", "JohnSnowLabs:spark-nlp:2.0.7,com.johnsnowlabs.nlp:spark-nlp-ocr_2.11:2.0.7,javax.media.jai:com.springsource.javax.media.jai.core:1.1.3")
         .config("spark.jars.repositories", "http://repo.spring.io/plugins-release")
     } else {
       build
-        .config("spark.jars.packages", "JohnSnowLabs:spark-nlp:2.0.6")
+        .config("spark.jars.packages", "JohnSnowLabs:spark-nlp:2.0.7")
     }
 
     build.getOrCreate()
   }
 
   def version(): Unit = {
-    println("2.0.6")
+    println("2.0.7")
   }
 
 }

src/main/scala/com/johnsnowlabs/util/Build.scala

@@ -11,6 +11,6 @@ object Build {
     if (version != null && version.nonEmpty)
       version
     else
-      "2.0.6"
+      "2.0.7"
   }
 }