Skip to content

Commit

Permalink
Merge pull request #13789 from JohnSnowLabs/release/442-release-candi…
Browse files Browse the repository at this point in the history 
…date

Release/442 release candidate
  • Loading branch information
@maziyarpanahi
maziyarpanahi authored May 10, 2023
2 parents b262eed + a99eaa2 commit f8354b3
Show file tree
Hide file tree
Showing 1,499 changed files with 35,317 additions and 7,693 deletions.
148 changes: 92 additions & 56 deletions .github/workflows/build_and_test.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,13 +31,13 @@ on:
- 'main'

jobs:
spark33:
spark34:
if: "! contains(toJSON(github.event.commits.*.message), '[skip test]')"
runs-on: macos-latest
env:
TF_CPP_MIN_LOG_LEVEL: 3
JAVA_OPTS: "-Xmx4096m -XX:+UseG1GC"
name: Build and Test on Apache Spark 3.3.x
name: Build and Test on Apache Spark 3.4.x

steps:
- uses: actions/checkout@v3
Expand All @@ -54,31 +54,31 @@ jobs:
- name: Install Python packages (Python 3.7)
run: |
python -m pip install --upgrade pip
pip install pyspark==3.3.1 numpy pytest
- name: Build Spark NLP on Apache Spark 3.3.0
pip install pyspark==3.4.0 numpy pytest
- name: Build Spark NLP on Apache Spark 3.4.0
run: |
brew install sbt
sbt -mem 4096 -Dis_spark33=true clean assemblyAndCopy
- name: Test Spark NLP in Scala - Apache Spark 3.3.x
sbt -mem 4096 -Dis_spark34=true clean assemblyAndCopy
- name: Test Spark NLP in Scala - Apache Spark 3.4.x
run: |
sbt -mem 4096 coverage test
- name: Upload coverage data to Coveralls
run: sbt coverageReport coveralls
env:
COVERALLS_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
COVERALLS_FLAG_NAME: Apache Spark 3.3.x - Scala 2.12
- name: Test Spark NLP in Python - Apache Spark 3.3.x
COVERALLS_FLAG_NAME: Apache Spark 3.4.x - Scala 2.12
- name: Test Spark NLP in Python - Apache Spark 3.4.x
run: |
cd python
python3.7 -m pytest -v -m fast
spark32:
spark33:
if: "! contains(toJSON(github.event.commits.*.message), '[skip test]')"
runs-on: macos-latest
env:
TF_CPP_MIN_LOG_LEVEL: 3
JAVA_OPTS: "-Xmx4096m -XX:+UseG1GC"
name: Build and Test on Apache Spark 3.2.x
name: Build and Test on Apache Spark 3.3.x

steps:
- uses: actions/checkout@v3
Expand All @@ -95,26 +95,26 @@ jobs:
- name: Install Python packages (Python 3.7)
run: |
python -m pip install --upgrade pip
pip install pyspark==3.2.3 numpy pytest
- name: Build Spark NLP on Apache Spark 3.2.3
pip install pyspark==3.3.1 numpy pytest
- name: Build Spark NLP on Apache Spark 3.3.1
run: |
brew install sbt
sbt -mem 4096 clean assemblyAndCopy
- name: Test Spark NLP in Scala - Apache Spark 3.2.x
sbt -mem 4096 -Dis_spark33=true clean assemblyAndCopy
- name: Test Spark NLP in Scala - Apache Spark 3.3.x
run: |
sbt -mem 4096 test
- name: Test Spark NLP in Python - Apache Spark 3.2.x
- name: Test Spark NLP in Python - Apache Spark 3.3.x
run: |
cd python
python3.7 -m pytest -v -m fast
spark31:
spark32:
if: "! contains(toJSON(github.event.commits.*.message), '[skip test]')"
runs-on: macos-latest
env:
TF_CPP_MIN_LOG_LEVEL: 3
JAVA_OPTS: "-Xmx4096m -XX:+UseG1GC"
name: Build and Test on Apache Spark 3.1.x
name: Build and Test on Apache Spark 3.2.x

steps:
- uses: actions/checkout@v3
Expand All @@ -131,51 +131,87 @@ jobs:
- name: Install Python packages (Python 3.7)
run: |
python -m pip install --upgrade pip
pip install pyspark==3.1.3 numpy pytest
- name: Build Spark NLP on Apache Spark 3.1.x
pip install pyspark==3.2.3 numpy pytest
- name: Build Spark NLP on Apache Spark 3.2.3
run: |
brew install sbt
sbt -mem 4096 -Dis_spark31=true clean assemblyAndCopy
- name: Test Spark NLP in Scala - Apache Spark 3.1.x
sbt -mem 4096 -Dis_spark32=true clean assemblyAndCopy
- name: Test Spark NLP in Scala - Apache Spark 3.2.x
run: |
sbt -mem 4096 test
- name: Test Spark NLP in Python - Apache Spark 3.1.x
- name: Test Spark NLP in Python - Apache Spark 3.2.x
run: |
cd python
python3.7 -m pytest -v -m fast
spark30:
if: "! contains(toJSON(github.event.commits.*.message), '[skip test]')"
runs-on: macos-latest
env:
TF_CPP_MIN_LOG_LEVEL: 3
JAVA_OPTS: "-Xmx4096m -XX:+UseG1GC"
name: Build and Test on Apache Spark 3.0.x
# spark31:
# if: "! contains(toJSON(github.event.commits.*.message), '[skip test]')"
# runs-on: macos-latest
# env:
# TF_CPP_MIN_LOG_LEVEL: 3
# JAVA_OPTS: "-Xmx4096m -XX:+UseG1GC"
# name: Build and Test on Apache Spark 3.1.x

steps:
- uses: actions/checkout@v3
- uses: actions/setup-java@v3
with:
distribution: 'adopt'
java-version: '8'
cache: 'sbt'
- name: Install Python 3.7
uses: actions/setup-python@v2
with:
python-version: 3.7.7
architecture: x64
- name: Install Python packages (Python 3.7)
run: |
python -m pip install --upgrade pip
pip install pyspark==3.0.3 numpy pytest
- name: Build Spark NLP on Apache Spark 3.0.x
run: |
brew install sbt
sbt -mem 4096 -Dis_spark30=true clean assemblyAndCopy
- name: Test Spark NLP in Scala - Apache Spark 3.0.x
run: |
sbt -mem 4096 test
- name: Test Spark NLP in Python - Apache Spark 3.0.x
run: |
cd python
python3.7 -m pytest -v -m fast
# steps:
# - uses: actions/checkout@v3
# - uses: actions/setup-java@v3
# with:
# distribution: 'adopt'
# java-version: '8'
# cache: 'sbt'
# - name: Install Python 3.7
# uses: actions/setup-python@v2
# with:
# python-version: 3.7.7
# architecture: x64
# - name: Install Python packages (Python 3.7)
# run: |
# python -m pip install --upgrade pip
# pip install pyspark==3.1.3 numpy pytest
# - name: Build Spark NLP on Apache Spark 3.1.x
# run: |
# brew install sbt
# sbt -mem 4096 -Dis_spark31=true clean assemblyAndCopy
# - name: Test Spark NLP in Scala - Apache Spark 3.1.x
# run: |
# sbt -mem 4096 test
# - name: Test Spark NLP in Python - Apache Spark 3.1.x
# run: |
# cd python
# python3.7 -m pytest -v -m fast

# spark30:
# if: "! contains(toJSON(github.event.commits.*.message), '[skip test]')"
# runs-on: macos-latest
# env:
# TF_CPP_MIN_LOG_LEVEL: 3
# JAVA_OPTS: "-Xmx4096m -XX:+UseG1GC"
# name: Build and Test on Apache Spark 3.0.x

# steps:
# - uses: actions/checkout@v3
# - uses: actions/setup-java@v3
# with:
# distribution: 'adopt'
# java-version: '8'
# cache: 'sbt'
# - name: Install Python 3.7
# uses: actions/setup-python@v2
# with:
# python-version: 3.7.7
# architecture: x64
# - name: Install Python packages (Python 3.7)
# run: |
# python -m pip install --upgrade pip
# pip install pyspark==3.0.3 numpy pytest
# - name: Build Spark NLP on Apache Spark 3.0.x
# run: |
# brew install sbt
# sbt -mem 4096 -Dis_spark30=true clean assemblyAndCopy
# - name: Test Spark NLP in Scala - Apache Spark 3.0.x
# run: |
# sbt -mem 4096 test
# - name: Test Spark NLP in Python - Apache Spark 3.0.x
# run: |
# cd python
# python3.7 -m pytest -v -m fast
20 changes: 20 additions & 0 deletions CHANGELOG
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,23 @@
========
4.4.2
========
----------------
New Features & Enhancements
----------------
* InImplement a new Zero-Shot Text Classification for RoBERTa annotator called `RobertaForZeroShotClassification`
* Support Apache Spark 3.4
* Omptize BART models for memory efficiency
* Introducing `cache` feature in BartTransformer
* Improve error handling for max sequence length for transformers in Python
* Improve `MultiDateMatcher` annotator to return multiple dates

----------------
Bug Fixes
----------------
* Fix a bug in Tapas due to exceeding the maximum rank value
* Fix loading Transformer models via loadSavedModel() method from DBFS on Databricks


========
4.4.1
========
Expand Down
Loading

0 comments on commit f8354b3

Please sign in to comment.