diff --git a/examples/python/annotation/text/english/DocumentCharacterTextSplitter.ipynb b/examples/python/annotation/text/english/DocumentCharacterTextSplitter.ipynb
new file mode 100644
index 00000000000000..8b07dd8452fc70
--- /dev/null
+++ b/examples/python/annotation/text/english/DocumentCharacterTextSplitter.ipynb
@@ -0,0 +1,379 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "97EiXueJA9cY"
+ },
+ "source": [
+ ""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "zmxL_blSA9ce"
+ },
+ "source": [
+ "[](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp/blob/master/examples/python/annotation/text/english/DocumentCharacterTextSplitter.ipynb)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "uI7yhCibA9cf"
+ },
+ "source": [
+ "## Colab + Data Setup"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "4WQLLrIUA9cg",
+ "outputId": "8db13c0f-d5c0-4af7-c08d-78d1ea820653"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Installing PySpark 3.2.3 and Spark NLP 5.2.2\n",
+ "setup Colab for PySpark 3.2.3 and Spark NLP 5.2.2\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m281.5/281.5 MB\u001b[0m \u001b[31m2.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m547.3/547.3 kB\u001b[0m \u001b[31m30.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m199.7/199.7 kB\u001b[0m \u001b[31m15.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[?25h Building wheel for pyspark (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
+ ]
+ }
+ ],
+ "source": [
+ "!wget -q http://setup.johnsnowlabs.com/colab.sh -O - | bash"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {
+ "id": "nVTDX8SdiSD9"
+ },
+ "outputs": [],
+ "source": [
+ "!wget https://github.com/JohnSnowLabs/spark-nlp/blob/587f79020de7bc09c2b2fceb37ec258bad57e425/src/test/resources/spell/sherlockholmes.txt > /dev/null 2>&1"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "KzMHa0HdA9ch",
+ "outputId": "f8f8b92a-12d5-45d9-a021-acd48241c4df"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Spark NLP version 5.2.2\n",
+ "Apache Spark version: 3.2.3\n"
+ ]
+ }
+ ],
+ "source": [
+ "import sparknlp\n",
+ "from sparknlp.base import *\n",
+ "from sparknlp.annotator import *\n",
+ "from pyspark.ml import Pipeline\n",
+ "\n",
+ "spark = sparknlp.start()\n",
+ "\n",
+ "print(f\"Spark NLP version {sparknlp.version()}\\nApache Spark version: {spark.version}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {
+ "id": "6qAa9p6ohtfi"
+ },
+ "outputs": [],
+ "source": [
+ "textDF = spark.read.text(\n",
+ " \"sherlockholmes.txt\",\n",
+ " wholetext=True\n",
+ ").toDF(\"text\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "_S-XJDfUA9ci"
+ },
+ "source": [
+ "# Download DocumentTokenSplitter Model and Create Spark NLP Pipeline"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "DVHludGFMSCk",
+ "outputId": "35716c76-f75f-4b53-b9c0-0235ec3d2db4"
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "sparknlp.annotator.document_character_text_splitter.DocumentCharacterTextSplitter"
+ ]
+ },
+ "execution_count": 5,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "DocumentCharacterTextSplitter"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "O4uPbdrSA9ci"
+ },
+ "source": [
+ "Lets create a Spark NLP pipeline with the following stages:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "ASQ5Ot2NA9ci",
+ "outputId": "eebfcf2d-8674-4cf7-8ece-b41de8f6bb0a"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "+--------------------------------------------------------------------------------+---------------+-------------+------+\n",
+ "| result|splits[0].begin|splits[0].end|length|\n",
+ "+--------------------------------------------------------------------------------+---------------+-------------+------+\n",
+ "|[{\"payload\":{\"allShortcutsEnabled\":false,\"fileTree\":{\"src/test/resources/spel...| 0| 19998| 19998|\n",
+ "|[Doctor, and give us your best\",\"attention.\\\"\",\"\",\"A slow and heavy step, whi...| 19806| 39805| 19999|\n",
+ "|[he said as he turned hungrily on the simple fare that\",\"our landlady had pro...| 39606| 59604| 19998|\n",
+ "|[armchair and\",\"putting his fingertips together, as was his custom when in\",\"...| 59407| 79406| 19999|\n",
+ "|[after a time, he did not come in at\",\"all. Still, of course, I never dared t...| 79208| 99201| 19993|\n",
+ "|[least an hour before us,\\\" he remarked, \\\"for they can\",\"hardly take any ste...| 99003| 119000| 19997|\n",
+ "|[after father's death, and\",\"a man who was nearly fifteen years younger than ...| 118801| 138782| 19981|\n",
+ "|[some of the details are of interest. The only drawback\",\"is that there is no...| 138599| 158591| 19992|\n",
+ "+--------------------------------------------------------------------------------+---------------+-------------+------+\n",
+ "only showing top 8 rows\n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "documentAssembler = DocumentAssembler() \\\n",
+ " .setInputCol(\"text\") \\\n",
+ " .setOutputCol(\"document\")\n",
+ "\n",
+ "textSplitter = DocumentCharacterTextSplitter() \\\n",
+ " .setInputCols([\"document\"]) \\\n",
+ " .setOutputCol(\"splits\") \\\n",
+ " .setChunkSize(20000) \\\n",
+ " .setChunkOverlap(200) \\\n",
+ " .setExplodeSplits(True)\n",
+ "\n",
+ "pipeline = Pipeline().setStages([documentAssembler, textSplitter])\n",
+ "result = pipeline.fit(textDF).transform(textDF)\n",
+ "\n",
+ "result.selectExpr(\n",
+ " \"splits.result\",\n",
+ " \"splits[0].begin\",\n",
+ " \"splits[0].end\",\n",
+ " \"splits[0].end - splits[0].begin as length\").show(8, truncate = 80)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "CALoU6tSofto"
+ },
+ "source": [
+ "# Now let's make another pipeline to see if this actually works!"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "H5DFx2DOosri"
+ },
+ "source": [
+ "let's get the data ready"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {
+ "id": "ZqR7pcQ9pw7a"
+ },
+ "outputs": [],
+ "source": [
+ "df = spark.createDataFrame([\n",
+ " [(\"All emotions, and that\\none particularly, were abhorrent to his cold, \"\n",
+ " \"precise but\\nadmirably balanced mind.\\n\\nHe was, I take it, the most \"\n",
+ " \"perfect\\nreasoning and observing machine that the world has seen.\")]\n",
+ "]).toDF(\"text\")\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "ArsOgKafoft0"
+ },
+ "source": [
+ "Lets create a Spark NLP pipeline following the same stages as before:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {
+ "id": "ph0Wm4qqLgVC"
+ },
+ "outputs": [],
+ "source": [
+ "documentAssembler = DocumentAssembler() \\\n",
+ " .setInputCol(\"text\") \\\n",
+ " .setOutputCol(\"document\")\n",
+ "\n",
+ "document_character_text_splitter = DocumentCharacterTextSplitter() \\\n",
+ " .setInputCols(\"document\") \\\n",
+ " .setOutputCol(\"splits\") \\\n",
+ " .setChunkSize(20) \\\n",
+ " .setChunkOverlap(5) \\\n",
+ " .setExplodeSplits(True) \\\n",
+ " .setPatternsAreRegex(False) \\\n",
+ " .setKeepSeparators(True) \\\n",
+ " .setSplitPatterns([\"\\n\\n\", \"\\n\", \" \", \"\"]) \\\n",
+ " .setTrimWhitespace(True)\n",
+ "\n",
+ "pipeline = Pipeline().setStages([documentAssembler, document_character_text_splitter])\n",
+ "pipeline_df = pipeline.fit(df).transform(df)\n",
+ "\n",
+ "results = pipeline_df.select(\"splits\").collect()\n",
+ "\n",
+ "splits = [\n",
+ " row[\"splits\"][0].result.replace(\"\\n\\n\", \" \").replace(\"\\n\", \" \")\n",
+ " for row in results\n",
+ "]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "mjUiY6sOp-jY"
+ },
+ "source": [
+ "**Evaluation**"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "S_P2TnL-MCqM",
+ "outputId": "35dcc363-2975-4ab3-cae0-47e10fba8f5f"
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "True"
+ ]
+ },
+ "execution_count": 15,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "expected = [\n",
+ " \"All emotions, and\",\n",
+ " \"and that\",\n",
+ " \"one particularly,\",\n",
+ " \"were abhorrent to\",\n",
+ " \"to his cold,\",\n",
+ " \"precise but\",\n",
+ " \"admirably balanced\",\n",
+ " \"mind.\",\n",
+ " \"He was, I take it,\",\n",
+ " \"it, the most\",\n",
+ " \"most perfect\",\n",
+ " \"reasoning and\",\n",
+ " \"and observing\",\n",
+ " \"machine that the\",\n",
+ " \"the world has seen.\",\n",
+ "]\n",
+ "\n",
+ "splits == expected"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "Wq4G03A2qB5U"
+ },
+ "source": [
+ "Great it works!"
+ ]
+ }
+ ],
+ "metadata": {
+ "colab": {
+ "collapsed_sections": [
+ "uI7yhCibA9cf"
+ ],
+ "provenance": []
+ },
+ "kernelspec": {
+ "display_name": "Python [conda env:tempspark]",
+ "language": "python",
+ "name": "conda-env-tempspark-py"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.8.16"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/examples/python/annotation/text/english/language-translation/Multilingual_Translation_with_M2M100.ipynb b/examples/python/annotation/text/english/language-translation/Multilingual_Translation_with_M2M100.ipynb
index 48d94f53211b24..d39125bd0ba2c9 100644
--- a/examples/python/annotation/text/english/language-translation/Multilingual_Translation_with_M2M100.ipynb
+++ b/examples/python/annotation/text/english/language-translation/Multilingual_Translation_with_M2M100.ipynb
@@ -8,17 +8,17 @@
"source": [
"\n",
"\n",
- "[](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp/tree/master/examples/python/annotation/text/english/language-translation/Multilingual_Translation_with_M2M100.ipynb)"
+ "[](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp/blob/master/examples/python/annotation/text/english/language-translation/Multilingual_Translation_with_M2M100.ipynb)"
]
},
{
"cell_type": "markdown",
- "source": [
- "# Multilingual Translation with M2M100"
- ],
"metadata": {
"id": "_b6aR_k6Oi9Q"
- }
+ },
+ "source": [
+ "# Multilingual Translation with M2M100"
+ ]
},
{
"cell_type": "markdown",
@@ -33,16 +33,16 @@
"cell_type": "code",
"execution_count": 1,
"metadata": {
- "id": "4WQLLrIUA9cg",
"colab": {
"base_uri": "https://localhost:8080/"
},
+ "id": "4WQLLrIUA9cg",
"outputId": "87c0ea9b-0f20-4361-fa2e-5eb6f8113507"
},
"outputs": [
{
- "output_type": "stream",
"name": "stdout",
+ "output_type": "stream",
"text": [
"Installing PySpark 3.2.3 and Spark NLP 5.3.0\n",
"setup Colab for PySpark 3.2.3 and Spark NLP 5.3.0\n",
@@ -62,16 +62,16 @@
"cell_type": "code",
"execution_count": 2,
"metadata": {
- "id": "KzMHa0HdA9ch",
"colab": {
"base_uri": "https://localhost:8080/"
},
+ "id": "KzMHa0HdA9ch",
"outputId": "f190dcf5-01bb-49d1-e17a-79a4123c3a13"
},
"outputs": [
{
- "output_type": "stream",
"name": "stdout",
+ "output_type": "stream",
"text": [
"Spark NLP version 5.3.0\n",
"Apache Spark version: 3.2.3\n"
@@ -123,16 +123,16 @@
"cell_type": "code",
"execution_count": 4,
"metadata": {
- "id": "0UNHPnccIxul",
- "outputId": "d0f00dcc-d7bc-480d-9e82-e924eace7d1a",
"colab": {
"base_uri": "https://localhost:8080/"
- }
+ },
+ "id": "0UNHPnccIxul",
+ "outputId": "d0f00dcc-d7bc-480d-9e82-e924eace7d1a"
},
"outputs": [
{
- "output_type": "stream",
"name": "stdout",
+ "output_type": "stream",
"text": [
"m2m100_418M download started this may take some time.\n",
"Approximate size to download 2.8 GB\n",
@@ -159,66 +159,59 @@
},
{
"cell_type": "markdown",
- "source": [
- "# Light Pipeline version"
- ],
"metadata": {
"id": "v4QtpYx2SK7y"
- }
+ },
+ "source": [
+ "# Light Pipeline version"
+ ]
},
{
"cell_type": "markdown",
- "source": [
- "Let's create the light Pipiline"
- ],
"metadata": {
"id": "XpcordqsSmE3"
- }
+ },
+ "source": [
+ "Let's create the light Pipiline"
+ ]
},
{
"cell_type": "code",
+ "execution_count": 7,
+ "metadata": {
+ "id": "CoQAJpOxQ1zb"
+ },
+ "outputs": [],
"source": [
"empty_df = spark.createDataFrame([[\"\"]]).toDF('text')\n",
"pipeline_model = tl_pipeline.fit(empty_df)\n",
"model = LightPipeline(pipeline_model)\n",
"res = model.fullAnnotate(text)"
- ],
- "metadata": {
- "id": "CoQAJpOxQ1zb"
- },
- "execution_count": 7,
- "outputs": []
+ ]
},
{
"cell_type": "markdown",
- "source": [
- "visualize the results"
- ],
"metadata": {
"id": "t-L_JARWSszu"
- }
+ },
+ "source": [
+ "visualize the results"
+ ]
},
{
"cell_type": "code",
- "source": [
- "print ('Original:', text, '\\n\\n')\n",
- "\n",
- "print ('Translated:\\n')\n",
- "for sentence in res[0]['generation']:\n",
- " print (sentence.result)"
- ],
+ "execution_count": 16,
"metadata": {
- "id": "ZnD-QGuJREW8",
"colab": {
"base_uri": "https://localhost:8080/"
},
+ "id": "ZnD-QGuJREW8",
"outputId": "51258f55-1ec2-4e86-f312-1e057dbec5b8"
},
- "execution_count": 16,
"outputs": [
{
- "output_type": "stream",
"name": "stdout",
+ "output_type": "stream",
"text": [
"Original: 除了是北方之王之外，约翰·斯诺还是一位英国医生，也是麻醉和医疗卫生发展的领导者。 他被认为是第一个利用数据治愈 1854 年霍乱爆发的人。 \n",
"\n",
@@ -228,6 +221,13 @@
"In addition to being the King of the North, John Snow was also a British doctor and a leader in the development of anesthesia and health care. he was considered the first person to use data to cure the 1854 cholera outbreak.\n"
]
}
+ ],
+ "source": [
+ "print ('Original:', text, '\\n\\n')\n",
+ "\n",
+ "print ('Translated:\\n')\n",
+ "for sentence in res[0]['generation']:\n",
+ " print (sentence.result)"
]
}
],
@@ -254,4 +254,4 @@
},
"nbformat": 4,
"nbformat_minor": 0
-}
\ No newline at end of file
+}
diff --git a/examples/python/annotation/text/english/question-answering/MPNetForQuestionAnswering.ipynb b/examples/python/annotation/text/english/question-answering/MPNetForQuestionAnswering.ipynb
new file mode 100644
index 00000000000000..fa36a1a15826a3
--- /dev/null
+++ b/examples/python/annotation/text/english/question-answering/MPNetForQuestionAnswering.ipynb
@@ -0,0 +1,379 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "97EiXueJA9cY"
+ },
+ "source": [
+ ""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "zmxL_blSA9ce"
+ },
+ "source": [
+ "[](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp/blob/master/examples/python/annotation/text/english/question-answering/MPNetForQuestionAnswering.ipynb)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "uI7yhCibA9cf"
+ },
+ "source": [
+ "## Colab Setup"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "4WQLLrIUA9cg",
+ "outputId": "d5760193-3cd6-45df-d354-def6d62b3c8a"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Installing PySpark 3.2.3 and Spark NLP 5.3.1\n",
+ "setup Colab for PySpark 3.2.3 and Spark NLP 5.3.1\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m281.5/281.5 MB\u001b[0m \u001b[31m4.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m564.8/564.8 kB\u001b[0m \u001b[31m42.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m199.7/199.7 kB\u001b[0m \u001b[31m19.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[?25h Building wheel for pyspark (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
+ ]
+ }
+ ],
+ "source": [
+ "! wget -q http://setup.johnsnowlabs.com/colab.sh -O - | bash"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "_S-XJDfUA9ci"
+ },
+ "source": [
+ "# Download MPNetForQuestionAnswering Model and Create Spark NLP Pipeline"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "O4uPbdrSA9ci"
+ },
+ "source": [
+ "Lets create a Spark NLP pipeline with the following stages:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "KzMHa0HdA9ch",
+ "outputId": "4c19dc18-abdc-4c87-f5e3-2924c5dae7af"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Spark NLP version 5.3.1\n",
+ "Apache Spark version: 3.2.3\n"
+ ]
+ }
+ ],
+ "source": [
+ "import sparknlp\n",
+ "from sparknlp.base import *\n",
+ "from sparknlp.common import *\n",
+ "from sparknlp.annotator import *\n",
+ "from pyspark.ml import Pipeline\n",
+ "import pandas as pd\n",
+ "\n",
+ "# for GPU training >> sparknlp.start(gpu = True)\n",
+ "spark = sparknlp.start()\n",
+ "\n",
+ "print(\"Spark NLP version\", sparknlp.version())\n",
+ "print(\"Apache Spark version:\", spark.version)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 43,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 205
+ },
+ "id": "DVHludGFMSCk",
+ "outputId": "fa69c6f0-d320-436a-b3e5-2021d73538bc"
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "
sparknlp.annotator.classifier_dl.mpnet_for_question_answering.MPNetForQuestionAnswering
def __init__(classname='com.johnsnowlabs.nlp.annotators.classifier.dl.MPNetForQuestionAnswering', java_model=None)
/usr/local/lib/python3.10/dist-packages/sparknlp/annotator/classifier_dl/mpnet_for_question_answering.pyMPNetForQuestionAnswering can load MPNet Models with a span classification head on top for extractive\n",
+ "question-answering tasks like SQuAD (a linear layer on top of the hidden-states output to compute span start\n",
+ "logits and span end logits).\n",
+ "\n",
+ "Pretrained models can be loaded with :meth:`.pretrained` of the companion\n",
+ "object:\n",
+ "\n",
+ ">>> spanClassifier = MPNetForQuestionAnswering.pretrained() \\\n",
+ "... .setInputCols(["document_question", "document_context"]) \\\n",
+ "... .setOutputCol("answer")\n",
+ "\n",
+ "The default model is ``"mpnet_base_question_answering_squad2"``, if no name is\n",
+ "provided.\n",
+ "\n",
+ "For available pretrained models please see the `Models Hub\n",
+ "<https://sparknlp.org/models?task=Question+Answering>`__.\n",
+ "\n",
+ "To see which models are compatible and how to import them see\n",
+ "`Import Transformers into Spark NLP 🚀\n",
+ "<https://github.com/JohnSnowLabs/spark-nlp/discussions/5669>`_.\n",
+ "\n",
+ "====================== ======================\n",
+ "Input Annotation types Output Annotation type\n",
+ "====================== ======================\n",
+ "``DOCUMENT, DOCUMENT`` ``CHUNK``\n",
+ "====================== ======================\n",
+ "\n",
+ "Parameters\n",
+ "----------\n",
+ "batchSize\n",
+ " Batch size. Large values allows faster processing but requires more\n",
+ " memory, by default 8\n",
+ "caseSensitive\n",
+ " Whether to ignore case in tokens for embeddings matching, by default\n",
+ " False\n",
+ "maxSentenceLength\n",
+ " Max sentence length to process, by default 128\n",
+ "\n",
+ "Examples\n",
+ "--------\n",
+ ">>> import sparknlp\n",
+ ">>> from sparknlp.base import *\n",
+ ">>> from sparknlp.annotator import *\n",
+ ">>> from pyspark.ml import Pipeline\n",
+ ">>> documentAssembler = MultiDocumentAssembler() \\\n",
+ "... .setInputCols(["question", "context"]) \\\n",
+ "... .setOutputCol(["document_question", "document_context"])\n",
+ ">>> spanClassifier = MPNetForQuestionAnswering.pretrained() \\\n",
+ "... .setInputCols(["document_question", "document_context"]) \\\n",
+ "... .setOutputCol("answer") \\\n",
+ "... .setCaseSensitive(False)\n",
+ ">>> pipeline = Pipeline().setStages([\n",
+ "... documentAssembler,\n",
+ "... spanClassifier\n",
+ "... ])\n",
+ ">>> data = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context")\n",
+ ">>> result = pipeline.fit(data).transform(data)\n",
+ ">>> result.select("answer.result").show(truncate=False)\n",
+ "+--------------------+\n",
+ "|result |\n",
+ "+--------------------+\n",
+ "|[Clara] |\n",
+ "+--------------------+\n",
+ " \n",
+ "
\n",
+ "
sparknlp.annotator.embeddings.bge_embeddings.BGEEmbeddings
def __init__(classname='com.johnsnowlabs.nlp.embeddings.BGEEmbeddings', java_model=None)
/usr/local/lib/python3.10/dist-packages/sparknlp/annotator/embeddings/bge_embeddings.pySentence embeddings using BGE.\n",
+ "\n",
+ " BGE, or BAAI General Embeddings, a model that can map any text to a low-dimensional dense \n",
+ "vector which can be used for tasks like retrieval, classification, clustering, or semantic search.\n",
+ "\n",
+ "Pretrained models can be loaded with `pretrained` of the companion object:\n",
+ "\n",
+ " >>> embeddings = BGEEmbeddings.pretrained() \\\n",
+ " ... .setInputCols(["document"]) \\\n",
+ " ... .setOutputCol("bge_embeddings")\n",
+ "\n",
+ "\n",
+ " The default model is ``"bge_base"``, if no name is provided.\n",
+ "\n",
+ " For available pretrained models please see the\n",
+ " `Models Hub <https://sparknlp.org/models?q=BGE>`__.\n",
+ "\n",
+ "\n",
+ " ====================== ======================\n",
+ " Input Annotation types Output Annotation type\n",
+ " ====================== ======================\n",
+ " ``DOCUMENT`` ``SENTENCE_EMBEDDINGS``\n",
+ " ====================== ======================\n",
+ "\n",
+ " Parameters\n",
+ " ----------\n",
+ " batchSize\n",
+ " Size of every batch , by default 8\n",
+ " dimension\n",
+ " Number of embedding dimensions, by default 768\n",
+ " caseSensitive\n",
+ " Whether to ignore case in tokens for embeddings matching, by default False\n",
+ " maxSentenceLength\n",
+ " Max sentence length to process, by default 512\n",
+ " configProtoBytes\n",
+ " ConfigProto from tensorflow, serialized into byte array.\n",
+ "\n",
+ " References\n",
+ " ----------\n",
+ " `C-Pack: Packaged Resources To Advance General Chinese Embedding <https://arxiv.org/pdf/2309.07597>`__\n",
+ " `BGE Github Repository <https://github.com/FlagOpen/FlagEmbedding>`__\n",
+ "\n",
+ " **Paper abstract**\n",
+ "\n",
+ " *We introduce C-Pack, a package of resources that significantly advance the field of general\n",
+ " Chinese embeddings. C-Pack includes three critical resources. \n",
+ " 1) C-MTEB is a comprehensive benchmark for Chinese text embeddings covering 6 tasks and 35 datasets.\n",
+ " 2) C-MTP is a massive text embedding dataset curated from labeled and unlabeled Chinese corpora\n",
+ " for training embedding models.\n",
+ " 3) C-TEM is a family of embedding models covering multiple sizes.\n",
+ " Our models outperform all prior Chinese text embeddings on C-MTEB by up to +10% upon the \n",
+ " time of the release. We also integrate and optimize the entire suite of training methods for\n",
+ " C-TEM. Along with our resources on general Chinese embedding, we release our data and models for\n",
+ " English text embeddings. The English models achieve stateof-the-art performance on the MTEB\n",
+ " benchmark; meanwhile, our released English data is 2 times larger than the Chinese data. All\n",
+ " these resources are made publicly available at https://github.com/FlagOpen/FlagEmbedding.*\n",
+ "\n",
+ " Examples\n",
+ " --------\n",
+ " >>> import sparknlp\n",
+ " >>> from sparknlp.base import *\n",
+ " >>> from sparknlp.annotator import *\n",
+ " >>> from pyspark.ml import Pipeline\n",
+ " >>> documentAssembler = DocumentAssembler() \\\n",
+ " ... .setInputCol("text") \\\n",
+ " ... .setOutputCol("document")\n",
+ " >>> embeddings = BGEEmbeddings.pretrained() \\\n",
+ " ... .setInputCols(["document"]) \\\n",
+ " ... .setOutputCol("bge_embeddings")\n",
+ " >>> embeddingsFinisher = EmbeddingsFinisher() \\\n",
+ " ... .setInputCols(["bge_embeddings"]) \\\n",
+ " ... .setOutputCols("finished_embeddings") \\\n",
+ " ... .setOutputAsVector(True)\n",
+ " >>> pipeline = Pipeline().setStages([\n",
+ " ... documentAssembler,\n",
+ " ... embeddings,\n",
+ " ... embeddingsFinisher\n",
+ " ... ])\n",
+ " >>> data = spark.createDataFrame([["query: how much protein should a female eat",\n",
+ " ... "passage: As a general guideline, the CDC's average requirement of protein for women ages 19 to 70 is 46 grams per day." + ... "But, as you can see from this chart, you'll need to increase that if you're expecting or training for a" + ... "marathon. Check out the chart below to see how much protein you should be eating each day.",\n",
+ " ... ]]).toDF("text")\n",
+ " >>> result = pipeline.fit(data).transform(data)\n",
+ " >>> result.selectExpr("explode(finished_embeddings) as result").show(5, 80)\n",
+ " +--------------------------------------------------------------------------------+\n",
+ " | result|\n",
+ " +--------------------------------------------------------------------------------+\n",
+ " |[[8.0190285E-4, -0.005974853, -0.072875895, 0.007944068, 0.026059335, -0.0080...|\n",
+ " |[[0.050514214, 0.010061974, -0.04340176, -0.020937217, 0.05170225, 0.01157857...|\n",
+ " +--------------------------------------------------------------------------------+\n",
+ " \n",
+ " \n",
+ "
\n",
+ "
sparknlp.annotator.embeddings.mpnet_embeddings.MPNetEmbeddings
def __init__(classname='com.johnsnowlabs.nlp.embeddings.MPNetEmbeddings', java_model=None)
/usr/local/lib/python3.10/dist-packages/sparknlp/annotator/embeddings/mpnet_embeddings.pySentence embeddings using MPNet.\n",
+ "\n",
+ "MPNet adopts a novel pre-training method, named masked and permuted language modeling,\n",
+ "to inherit the advantages of masked language modeling and permuted language modeling for\n",
+ "natural language understanding.\n",
+ "\n",
+ "Pretrained models can be loaded with :meth:`.pretrained` of the companion\n",
+ "object:\n",
+ "\n",
+ ">>> embeddings = MPNetEmbeddings.pretrained() \\\n",
+ "... .setInputCols(["document"]) \\\n",
+ "... .setOutputCol("mpnet_embeddings")\n",
+ "\n",
+ "\n",
+ "The default model is ``"all_mpnet_base_v2"``, if no name is provided.\n",
+ "\n",
+ "For available pretrained models please see the\n",
+ "`Models Hub <https://sparknlp.org/models?q=MPNet>`__.\n",
+ "\n",
+ "\n",
+ "====================== ======================\n",
+ "Input Annotation types Output Annotation type\n",
+ "====================== ======================\n",
+ "``DOCUMENT`` ``SENTENCE_EMBEDDINGS``\n",
+ "====================== ======================\n",
+ "\n",
+ "Parameters\n",
+ "----------\n",
+ "batchSize\n",
+ " Size of every batch , by default 8\n",
+ "dimension\n",
+ " Number of embedding dimensions, by default 768\n",
+ "caseSensitive\n",
+ " Whether to ignore case in tokens for embeddings matching, by default False\n",
+ "maxSentenceLength\n",
+ " Max sentence length to process, by default 512\n",
+ "configProtoBytes\n",
+ " ConfigProto from tensorflow, serialized into byte array.\n",
+ "\n",
+ "References\n",
+ "----------\n",
+ "`MPNet: Masked and Permuted Pre-training for Language Understanding <https://arxiv.org/pdf/2004.09297>`__\n",
+ "\n",
+ "https://github.com/microsoft/MPNet\n",
+ "\n",
+ "**Paper abstract**\n",
+ "\n",
+ "*BERT adopts masked language modeling (MLM) for pre-training and is one of the most successful pre-training models.\n",
+ " Since BERT neglects dependency among predicted tokens, XLNet introduces permuted language modeling (PLM) for\n",
+ " pre-training to address this problem. However, XLNet does not leverage the full position information of a sentence\n",
+ " and thus suffers from position discrepancy between pre-training and fine-tuning. In this paper, we propose MPNet,\n",
+ " a novel pre-training method that inherits the advantages of BERT and XLNet and avoids their limitations. MPNet\n",
+ " leverages the dependency among predicted tokens through permuted language modeling (vs. MLM in BERT), and takes\n",
+ " auxiliary position information as input to make the model see a full sentence and thus reducing the position\n",
+ " discrepancy (vs. PLM in XLNet). We pre-train MPNet on a large-scale dataset (over 160GB text corpora) and fine-tune\n",
+ " on a variety of down-streaming tasks (GLUE, SQuAD, etc). Experimental results show that MPNet outperforms MLM and\n",
+ " PLM by a large margin, and achieves better results on these tasks compared with previous state-of-the-art\n",
+ " pre-trained methods (e.g., BERT, XLNet, RoBERTa) under the same model setting.*\n",
+ "\n",
+ "Examples\n",
+ "--------\n",
+ ">>> import sparknlp\n",
+ ">>> from sparknlp.base import *\n",
+ ">>> from sparknlp.annotator import *\n",
+ ">>> from pyspark.ml import Pipeline\n",
+ ">>> documentAssembler = DocumentAssembler() \\\n",
+ "... .setInputCol("text") \\\n",
+ "... .setOutputCol("document")\n",
+ ">>> embeddings = MPNetEmbeddings.pretrained() \\\n",
+ "... .setInputCols(["document"]) \\\n",
+ "... .setOutputCol("mpnet_embeddings")\n",
+ ">>> embeddingsFinisher = EmbeddingsFinisher() \\\n",
+ "... .setInputCols(["mpnet_embeddings"]) \\\n",
+ "... .setOutputCols("finished_embeddings") \\\n",
+ "... .setOutputAsVector(True)\n",
+ ">>> pipeline = Pipeline().setStages([\n",
+ "... documentAssembler,\n",
+ "... embeddings,\n",
+ "... embeddingsFinisher\n",
+ "... ])\n",
+ ">>> data = spark.createDataFrame([["This is an example sentence", "Each sentence is converted"]]).toDF("text")\n",
+ ">>> result = pipeline.fit(data).transform(data)\n",
+ ">>> result.selectExpr("explode(finished_embeddings) as result").show(5, 80)\n",
+ "+--------------------------------------------------------------------------------+\n",
+ "| result|\n",
+ "+--------------------------------------------------------------------------------+\n",
+ "|[[0.022502584, -0.078291744, -0.023030775, -0.0051000593, -0.080340415, 0.039...|\n",
+ "|[[0.041702367, 0.0010974605, -0.015534201, 0.07092203, -0.0017729357, 0.04661...|\n",
+ "+--------------------------------------------------------------------------------+\n",
+ " \n",
+ "
\n",
+ "
sparknlp.annotator.classifier_dl.mpnet_for_sequence_classification.MPNetForSequenceClassification
def __init__(classname='com.johnsnowlabs.nlp.annotators.classifier.dl.MPNetForSequenceClassification', java_model=None)
/usr/local/lib/python3.10/dist-packages/sparknlp/annotator/classifier_dl/mpnet_for_sequence_classification.pyMPNetForSequenceClassification can load MPNet Models with sequence classification/regression head on\n",
+ "top (a linear layer on top of the pooled output) e.g. for multi-class document classification tasks.\n",
+ "\n",
+ "Pretrained models can be loaded with :meth:`.pretrained` of the companion\n",
+ "object:\n",
+ "\n",
+ ">>> sequenceClassifier = MPNetForSequenceClassification.pretrained() \\\n",
+ "... .setInputCols(["token", "document"]) \\\n",
+ "... .setOutputCol("label")\n",
+ "\n",
+ "The default model is ``"mpnet_sequence_classifier_ukr_message"``, if no name is\n",
+ "provided.\n",
+ "\n",
+ "For available pretrained models please see the `Models Hub\n",
+ "<https://sparknlp.org/models?task=Text+Classification>`__.\n",
+ "\n",
+ "To see which models are compatible and how to import them see\n",
+ "`Import Transformers into Spark NLP 🚀\n",
+ "<https://github.com/JohnSnowLabs/spark-nlp/discussions/5669>`_.\n",
+ "\n",
+ "====================== ======================\n",
+ "Input Annotation types Output Annotation type\n",
+ "====================== ======================\n",
+ "``DOCUMENT, TOKEN`` ``CATEGORY``\n",
+ "====================== ======================\n",
+ "\n",
+ "Parameters\n",
+ "----------\n",
+ "batchSize\n",
+ " Batch size. Large values allows faster processing but requires more\n",
+ " memory, by default 8\n",
+ "caseSensitive\n",
+ " Whether to ignore case in tokens for embeddings matching, by default\n",
+ " True\n",
+ "maxSentenceLength\n",
+ " Max sentence length to process, by default 128\n",
+ "coalesceSentences\n",
+ " Instead of 1 class per sentence (if inputCols is `sentence`) output\n",
+ " 1 class per document by averaging probabilities in all sentences, by\n",
+ " default False.\n",
+ "activation\n",
+ " Whether to calculate logits via Softmax or Sigmoid, by default\n",
+ " `"softmax"`.\n",
+ "\n",
+ "Examples\n",
+ "--------\n",
+ ">>> import sparknlp\n",
+ ">>> from sparknlp.base import *\n",
+ ">>> from sparknlp.annotator import *\n",
+ ">>> from pyspark.ml import Pipeline\n",
+ ">>> document = DocumentAssembler() \\\n",
+ "... .setInputCol("text") \\\n",
+ "... .setOutputCol("document")\n",
+ ">>> tokenizer = Tokenizer() \\\n",
+ "... .setInputCols(["document"]) \\\n",
+ "... .setOutputCol("token")\n",
+ ">>> sequenceClassifier = MPNetForSequenceClassification \\\n",
+ "... .pretrained() \\\n",
+ "... .setInputCols(["document", "token"]) \\\n",
+ "... .setOutputCol("label")\n",
+ ">>> data = spark.createDataFrame([\n",
+ "... ["I love driving my car."],\n",
+ "... ["The next bus will arrive in 20 minutes."],\n",
+ "... ["pineapple on pizza is the worst 🤮"],\n",
+ "... ]).toDF("text")\n",
+ ">>> pipeline = Pipeline().setStages([document, tokenizer, sequenceClassifier])\n",
+ ">>> pipelineModel = pipeline.fit(data)\n",
+ ">>> results = pipelineModel.transform(data)\n",
+ ">>> results.select("label.result").show()\n",
+ "+--------------------+\n",
+ "| result|\n",
+ "+--------------------+\n",
+ "| [TRANSPORT/CAR]|\n",
+ "|[TRANSPORT/MOVEMENT]|\n",
+ "| [FOOD]|\n",
+ "+--------------------+\n",
+ " \n",
+ "
\n",
+ "
sparknlp.annotator.classifier_dl.deberta_for_zero_shot_classification.DeBertaForZeroShotClassification
def __init__(classname='com.johnsnowlabs.nlp.annotators.classifier.dl.DeBertaForZeroShotClassification', java_model=None)
/usr/local/lib/python3.10/dist-packages/sparknlp/annotator/classifier_dl/deberta_for_zero_shot_classification.pyDeBertaForZeroShotClassification using a `ModelForSequenceClassification` trained on NLI (natural language\n",
+ "inference) tasks. Equivalent of `DeBertaForSequenceClassification` models, but these models don't require a hardcoded\n",
+ "number of potential classes, they can be chosen at runtime. It usually means it's slower but it is much more\n",
+ "flexible.\n",
+ "Any combination of sequences and labels can be passed and each combination will be posed as a premise/hypothesis\n",
+ "pair and passed to the pretrained model.\n",
+ "Pretrained models can be loaded with :meth:`.pretrained` of the companion\n",
+ "object:\n",
+ ">>> sequenceClassifier = DeBertaForZeroShotClassification.pretrained() \\\n",
+ "... .setInputCols(["token", "document"]) \\\n",
+ "... .setOutputCol("label")\n",
+ "The default model is ``"deberta_base_zero_shot_classifier_mnli_anli_v3"``, if no name is\n",
+ "provided.\n",
+ "For available pretrained models please see the `Models Hub\n",
+ "<https://sparknlp.orgtask=Text+Classification>`__.\n",
+ "To see which models are compatible and how to import them see\n",
+ "`Import Transformers into Spark NLP 🚀\n",
+ "<https://github.com/JohnSnowLabs/spark-nlp/discussions/5669>`_.\n",
+ "====================== ======================\n",
+ "Input Annotation types Output Annotation type\n",
+ "====================== ======================\n",
+ "``DOCUMENT, TOKEN`` ``CATEGORY``\n",
+ "====================== ======================\n",
+ "Parameters\n",
+ "----------\n",
+ "batchSize\n",
+ " Batch size. Large values allows faster processing but requires more\n",
+ " memory, by default 8\n",
+ "caseSensitive\n",
+ " Whether to ignore case in tokens for embeddings matching, by default\n",
+ " True\n",
+ "configProtoBytes\n",
+ " ConfigProto from tensorflow, serialized into byte array.\n",
+ "maxSentenceLength\n",
+ " Max sentence length to process, by default 128\n",
+ "coalesceSentences\n",
+ " Instead of 1 class per sentence (if inputCols is `sentence`) output 1\n",
+ " class per document by averaging probabilities in all sentences, by\n",
+ " default False\n",
+ "activation\n",
+ " Whether to calculate logits via Softmax or Sigmoid, by default\n",
+ " `"softmax"`.\n",
+ "Examples\n",
+ "--------\n",
+ ">>> import sparknlp\n",
+ ">>> from sparknlp.base import *\n",
+ ">>> from sparknlp.annotator import *\n",
+ ">>> from pyspark.ml import Pipeline\n",
+ ">>> documentAssembler = DocumentAssembler() \\\n",
+ "... .setInputCol("text") \\\n",
+ "... .setOutputCol("document")\n",
+ ">>> tokenizer = Tokenizer() \\\n",
+ "... .setInputCols(["document"]) \\\n",
+ "... .setOutputCol("token")\n",
+ ">>> sequenceClassifier = DeBertaForZeroShotClassification.pretrained() \\\n",
+ "... .setInputCols(["token", "document"]) \\\n",
+ "... .setOutputCol("label") \\\n",
+ "... .setCaseSensitive(True)\n",
+ ">>> pipeline = Pipeline().setStages([\n",
+ "... documentAssembler,\n",
+ "... tokenizer,\n",
+ "... sequenceClassifier\n",
+ "... ])\n",
+ ">>> data = spark.createDataFrame([["I loved this movie when I was a child.", "It was pretty boring."]]).toDF("text")\n",
+ ">>> result = pipeline.fit(data).transform(data)\n",
+ ">>> result.select("label.result").show(truncate=False)\n",
+ "+------+\n",
+ "|result|\n",
+ "+------+\n",
+ "|[pos] |\n",
+ "|[neg] |\n",
+ "+------+\n",
+ " \n",
+ "