examples/python/annotation/text/english/sentence-embeddings/InstructorEmbeddings.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "78c1b739",
   "metadata": {},
   "source": [
    "![JohnSnowLabs](https://nlp.johnsnowlabs.com/assets/images/logo.png)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp/blob/master/examples/python/annotation/text/english/sentence-embeddings/InstructorEmbeddings.ipynb)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Colab Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!wget http://setup.johnsnowlabs.com/colab.sh -O - | bash"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sparknlp\n",
    "\n",
    "from sparknlp.base import *\n",
    "from sparknlp.common import *\n",
    "from sparknlp.annotator import *\n",
    "from pyspark.ml import Pipeline\n",
    "import pandas as pd\n",
    "\n",
    "# Comment out this line  and uncomment the next one to enable GPU mode and High RAM\n",
    "# \n",
    "spark = sparknlp.start()\n",
    "\n",
    "# spark = sparknlp.start(gpu=True)\n",
    "\n",
    "print(\"Spark NLP version\", sparknlp.version())\n",
    "print(\"Apache Spark version:\", spark.version)\n",
    "\n",
    "spark"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "e9fa6df8",
   "metadata": {},
   "source": [
    "# Download InstructorEmbedding Model and Create Spark NLP Pipeline"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "efef6c46",
   "metadata": {},
   "source": [
    "Lets create a Spark NLP pipeline with the following stages:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3a08d21e-a84a-433b-91e2-4b98e841f4bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "document_assembler = DocumentAssembler() \\\n",
    "            .setInputCol(\"text\") \\\n",
    "            .setOutputCol(\"documents\")\n",
    "\n",
    "instruction = InstructorEmbeddings.pretrained(name='instructor_base', lang='en') \\\n",
    "            .setInstruction(\"Represent the Wikipedia document for retrieval: \") \\\n",
    "            .setInputCols([\"documents\"]) \\\n",
    "            .setOutputCol(\"instructor\")\n",
    "\n",
    "# Build pipeline with BART\n",
    "pipe_components = [document_assembler, instruction]\n",
    "pipeline = Pipeline().setStages( pipe_components)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "f3b0c64e",
   "metadata": {},
   "source": [
    "Lets create a dataframe with some text to be annotated."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "06c931fb-9acc-41ba-a1b5-944732e55aa9",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = spark.createDataFrame([\n",
    "            [1, \"\"\"Capitalism has been dominant in the Western world since the end of feudalism, but most feel[who?] that \n",
    "            the term \"mixed economies\" more precisely describes most contemporary economies, due to their containing both \n",
    "            private-owned and state-owned enterprises. In capitalism, prices determine the demand-supply scale. For \n",
    "            example, higher demand for certain goods and services lead to higher prices and lower demand for certain \n",
    "            goods lead to lower prices. \"\"\"]]).toDF(\"id\", \"text\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "cb66bf86",
   "metadata": {},
   "source": [
    "Extract the embeddings from the embeddings layer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "95dd8de4-3840-408a-b08f-3b10cd47faa9",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 0:>                                                          (0 + 1) / 1]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/06/07 12:28:56 WARN MemoryStore: Not enough space to cache broadcast_1 in memory! (computed 4.6 GiB so far)\n",
      "+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
      "|embeddings                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     |\n",
      "+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
      "|[[0.03262866, -0.02278552, 0.0013627451, -0.05349423, -0.086116955, -5.929518E-4, -0.0746425, -0.044477966, -0.030355768, -0.045733023, 0.05812091, 0.03179538, -0.04334866, -0.08494021, -0.035193942, 0.024432851, -4.2305558E-4, 0.007649129, 0.03761272, -0.015318542, 0.006661357, -0.018444803, -0.020876005, 0.055248085, -0.06265088, -0.07534279, 0.056617893, -0.004320507, -0.0316281, 0.018066075, 0.03855368, 0.0048050773, 0.028971815, -0.006877347, 0.07095821, 9.4414165E-4, 0.008437616, -0.027222378, 0.019745044, -0.027784333, -0.01716024, 0.073865384, 0.014833191, -0.022548206, 0.046016835, -0.011504175, 0.026386736, -0.050758336, 0.04021454, 0.008387009, 0.010211223, -0.039326932, -0.0030433831, 0.0046180817, 0.028014615, -0.016920675, -0.042716026, -0.02837658, -0.035181597, 0.06269657, 0.0174563, 0.0054392195, -0.012997994, -0.012604179, 0.04189872, 0.06422939, 0.021105042, 0.004485108, -0.04081728, 0.02868578, 4.2648325E-4, -0.0024915151, -0.021758735, -0.008838901, -0.024631336, 0.038712267, 0.048298344, 0.029415213, 0.04060175, 0.0037318747, -0.07376663, 0.009977012, -0.0369224, 0.03830379, 0.023158496, -0.006534989, -0.022895986, -0.021891661, 0.009341288, 0.051562686, -0.041494038, -0.06153403, -0.022601144, -0.02245917, -0.01168978, 0.015870133, 0.037891928, -0.009542872, -0.020980844, -0.009183635, -0.012983468, 0.05018471, -0.0018250593, -0.070793994, -0.030277211, -0.02960738, 0.03202237, 0.06728596, -0.021340372, 0.044753116, 0.05349561, 0.054893903, 0.010459065, -0.038412116, 0.032823145, -0.04889416, 0.07951994, 0.009052965, -0.038682017, 0.104713686, 0.06529797, 0.03164878, -0.031003132, -0.018868608, -0.017530592, 0.013969608, 0.03172863, -0.0062067593, 0.0027590087, 8.379946E-4, 0.038555548, 0.01284134, 0.05865323, -0.020161793, 0.037459202, -0.08371534, -0.0065423665, -0.032452952, -0.055465214, -0.08255787, 0.027767248, -0.026513763, -0.004781667, 0.010080541, -0.065526746, 0.01758705, -0.01110518, 0.008830303, 0.018352255, 0.050216753, -0.044997294, -0.024659796, 0.010087487, -0.010251949, 0.01201215, -0.01594586, -0.03205041, 0.024571417, 0.041641567, -0.05499973, 0.05176098, -0.0125949895, -0.10548955, 0.029421275, -0.08137917, 0.03397447, -0.024931302, -0.009527485, -0.027410189, 0.02738372, 0.029404989, 0.020453911, 0.025400365, 0.0026869376, 0.048473097, -0.04090833, 0.0032688805, 0.078267716, 0.058016837, -0.015349757, -0.02964962, 7.428633E-4, 0.07016278, 0.062107064, 0.0016332245, -0.0023410108, 0.014560618, 2.6762972E-4, 0.025807912, 0.026444439, 0.018800316, -0.090043195, 0.008166241, 0.04428991, -0.0136914905, -0.014657232, -0.021842893, 0.07175007, 0.0054828576, 0.0033373928, 0.030236384, 0.011349291, 0.023523187, 0.026469443, -0.011958254, 0.018875133, 0.011467337, 0.011527503, -0.014620561, 0.014909634, -0.03518385, 0.08208324, 0.024886962, -0.015513791, -0.038096245, 0.039179526, 0.034246866, -0.028717602, -0.025548302, 0.0368702, 0.007931793, 0.026460685, -0.00364875, -0.010761328, 0.054106724, -0.039791096, -0.07631004, 0.038830508, 0.06268836, -0.025078528, -0.026704583, -0.016394533, 0.03193358, 0.036612213, -0.024061156, -0.025402015, -4.0322175E-4, -0.062252726, -0.059745993, 0.008429678, 0.03852402, -0.055612803, -0.012058595, -0.044576805, 0.005232973, 0.007215015, -0.04405842, -0.022040177, 6.000543E-4, -0.032302897, -0.033665285, -0.008454371, 0.023602277, 0.006704567, -0.033100456, -0.01915944, -0.029278649, -0.027450416, 0.007982758, 0.02155662, -0.06519412, -0.020685513, 0.036739424, -0.019138498, 0.007845404, -0.0036733786, 0.03498778, -0.008669832, 0.032490708, 0.021262223, -0.020452157, 0.042844422, 0.039352544, -0.02001723, 0.0046986653, 0.039409067, -0.01183346, -0.034159135, -0.032669347, 0.034656446, -0.0026955863, -0.058706068, -0.014528975, 0.011225362, -0.018192712, 0.024715172, -0.013345554, -0.06291966, 0.028073484, -0.020382162, 0.003955219, 0.051585477, -0.059672885, -0.028786313, -0.049310677, 0.022147192, -0.016901385, 0.043830827, 0.052221525, -0.007187432, 0.0077309003, 0.022526953, -0.015928626, -0.058199972, -0.021536296, -0.00266462, 0.021913854, 0.025801338, -0.07448505, -0.01068346, 0.026055792, 0.003932057, 0.030646225, 0.020291511, -0.022222407, -0.049949016, 0.034922905, 0.031354185, 0.0044451198, 0.033286795, 0.047622625, -0.039508287, 0.016014116, -0.012728955, -0.014505607, 0.051102817, 0.05280988, 0.043907918, 0.037854925, -0.077935375, -0.028005999, 0.05174665, -0.025033204, 0.028977908, 0.033971164, 0.080860905, -0.011379051, -6.125111E-4, 0.015531941, 0.029136108, -0.043808825, 0.021221649, -0.0036628984, -0.014902094, -0.022289492, -0.06090777, 0.045519885, 0.049842637, -0.03771459, -0.044942044, -0.0059738797, 0.008993613, 0.047185607, 0.03366262, 0.04698031, -0.033994306, 0.028037608, 0.015821587, -0.006695826, 0.010477219, 0.04892719, 0.01928435, 0.009722461, -0.009907463, 0.008717666, 0.022839326, -0.064767726, -0.08677013, 0.05109122, 0.06738159, 0.026279185, 0.017919093, 0.034425717, 0.012025932, -0.017000718, 0.01568243, -0.043481626, -0.066099755, -0.036917772, -0.053656776, -0.0073990393, -0.015794588, 0.05056942, 0.041876696, 0.08926835, 0.021864763, 0.008828377, -0.082242414, 0.0016147506, 0.01908765, 0.021238945, 0.01666622, 0.0137727065, 0.033908978, 0.008618177, -0.0415076, -0.02719992, 0.004980526, -0.013505137, -0.032252524, 0.027290987, 1.2833111E-4, 0.052202977, 0.05350614, 0.004707537, -0.001970308, -0.014455575, 0.029612198, -0.013719515, -0.013465883, -0.005294823, -0.037157647, -0.032241847, 0.010828067, 0.015217094, 0.010721237, 0.0109178, 0.023587238, 0.03139799, 0.028548352, 0.007332915, -0.034978863, 0.04591757, 0.07023253, -0.037546985, -0.0057449825, 0.01028897, 0.024115201, -0.05835967, 0.015728861, 0.007577405, 0.066445865, -0.0109921, 0.009578472, -0.018897576, 0.03284056, -0.0078084227, 0.022518951, 0.005147412, 0.038542423, -0.037588574, 0.01890949, -0.009531408, 0.0743924, 0.03420677, -0.00388404, 0.016388329, -0.05744358, 0.028442338, 0.022417674, 0.018632138, 0.058668043, -0.043136194, 0.02837622, 0.066264585, 0.031632416, 0.009375166, -0.021616708, -0.046904657, -0.011571442, -0.026009535, 0.005370311, 0.0013603637, -0.0054784706, -0.015194243, -0.040979613, -0.011726019, 0.041559637, 0.017277045, 0.009240102, -0.02853778, 0.05799228, -0.013103851, -0.081491776, -0.05242484, -0.0498567, -0.011498945, -0.030589731, -0.06344852, -0.039167684, 0.037541926, 0.025322966, 0.03585917, -0.06897881, 0.013983291, 0.07146157, -0.058129825, -0.012389879, 0.044209927, 0.003640095, -0.03880821, -0.0061029145, 0.024704812, 0.0130145475, -0.008639259, -0.07097213, -0.037620023, -0.06390397, 0.012232612, -0.05413426, -0.026953025, -0.008528179, 7.501531E-4, 0.06066094, -8.8505604E-4, 0.015460086, 0.0061789826, 0.02452416, -0.021089591, 0.042983003, 0.0565328, 0.049946148, 0.0053586145, 0.01441645, -0.014378864, 0.0024458088, 0.01483357, 0.031564116, 0.0065488485, -0.027081812, -0.072921626, -0.028170047, 0.054090757, 0.02424426, -0.005840947, 0.0025784718, -0.008157144, -0.06483979, -0.0625094, 0.07688747, -0.016405731, -0.0011064977, -0.046268202, -0.025038263, 0.029777573, -3.960068E-4, 0.018920312, -0.022698821, -0.056359243, -0.035240088, -0.054945026, 0.060251184, -0.007809836, 0.032234326, -0.011087703, -0.018321337, 0.018414224, -0.040390566, 0.024843065, -0.0031665978, 0.07884546, -0.0035033596, -0.05727815, 0.043211542, -0.008092797, 0.08816403, 0.040669575, 0.0035789618, -0.0021728065, -0.034989733, -0.06114682, 0.015779695, -0.05540598, 0.027801292, -0.055221975, -0.014599996, 0.008345163, -0.054678977, 0.0057505197, 0.009509776, 0.086219944, 0.0027056169, -0.012578616, -0.019075787, 0.06411902, -0.0015007671, 0.024660196, -0.02524281, 0.018707322, 1.7817405E-5, -0.0067316075, -0.07148548, -0.038645077, -0.074731976, 0.06418082, -0.0025319012, -0.024625022, 0.042892747, -0.015663806, 0.009740704, -0.028613178, 0.02524035, -0.025803398, -0.004221928, -0.035811353, 0.03623488, 0.05823448, -0.011964531, -0.006890309, -0.05232545, -0.021972118, 0.0039357906, -0.004678495, 0.023032501, 0.049222905, 0.05967132, 0.02711187, -0.028429024, 0.0597026, 0.008744213, 0.039600607, -0.01799164, 0.029899204, 0.04086879, -0.007106043, -0.024120256, -0.045608405, 0.02578875, 0.034966342, 0.051028572, -0.04749465, 0.050027035, -0.026982184, 0.023817034, -0.005344621, 0.033817418, -0.04080848, -0.04403071, -0.052517183, -0.0038946117, -0.033082206, 0.004636815, 0.026521523, -0.026402375, -0.002364965, -8.322088E-4, -0.0243335, 0.07126975, 0.009148769, 0.034467135, 0.012993687, -0.010860814, -0.044525325, 0.07306212, -0.048434332, -0.07555003, 0.03128192, -0.021581138, -0.05958449, 0.024803905, -0.006846708, -0.027540546, -0.02732654, -0.070398346, -0.018436609, -0.024355149, -0.027597638, 0.03856409, -0.021519573, -0.07238803, -0.087142274, 0.01989241, -0.02820587, 0.009047604, 0.036062315, -0.0041471967, -0.07270885, 0.018391611, 0.011368535, 0.030430099, -0.05325456, -0.030990591, -0.017405782, 0.0051884362, 0.008637634, 0.045851987, 0.010309327, -0.036312383, -0.026670815, -0.01030423, 0.00986426, -0.0049678143, 0.009745687, 0.0062699416, -0.02588413, 0.0044184714, 0.011547122, 0.026856875, 0.0025859745, 0.007370621, 0.009339734, -0.034624025, 0.015941104, -0.026927661, 0.014354207, -0.025712667, 0.015726479, 0.049310084, -0.050456293, -0.04896244, -0.0038412337, -0.018047825, 0.047674045, -0.064386874, 0.018670822, -0.026679922, 0.0319397, -0.024611896, 0.022062775, 0.032022644, -0.063417435, -0.012273067, 0.011973682, 6.4346654E-4, 0.0041928133, -0.018408531, -0.011742022, 0.009397221, 0.054278865, 0.043511264, -0.0127404295, -1.7254091E-4, -0.0070622903, -0.0041349228, -0.019627219, 0.023186563, -0.023316137, -0.06182324, -0.003907854, -2.5893108E-4, 0.06966795, -0.018856877, 3.508146E-4, 0.004480817, -0.0029769759, 0.024018057, 0.065449946, 0.044452745, 0.013281421, 0.047196712, 0.027858423, 0.007940354, 0.033436798, -0.019692471, 0.06342478, -0.021618921, -0.03593985, 0.02641698, -0.05189184, -0.043362506, -0.050688393, -0.005378341, 0.0033666724, 0.044950858, 0.02093605, 0.03626855, -0.024815692, 0.028359616, 0.021613698, -0.020997578, 0.0027879598, -0.017473606, 0.017178757, -0.030661179, -0.013124452, -0.01470106, -0.03801807, 0.053062066]]|\n",
      "+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "results = pipeline.fit(data).transform(data)\n",
    "results.select(\"instructor.embeddings\").show(truncate=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:tempspark]",
   "language": "python",
   "name": "conda-env-tempspark-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}