Merge branch 'resource_fix' of https://github.com/JohnSnowLabs/spark-nlp

into resource_fix
JohnSnowLabs · Nov 14, 2017 · 9281af3 · 9281af3
2 parents 7c00d07 + e04bfb7
commit 9281af3
Show file tree

Hide file tree

Showing 2 changed files with 29 additions and 15 deletions.
diff --git a/python/example/crf-ner/ner.ipynb b/python/example/crf-ner/ner.ipynb
@@ -22,6 +22,24 @@
   {
    "cell_type": "code",
    "execution_count": 2,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "# Download CoNLL 2003 Dataset\n",
+    "import os\n",
+    "from pathlib import Path\n",
+    "import urllib.request\n",
+    "\n",
+    "if not Path(\"eng.train\").is_file():\n",
+    "    url = \"https://github.com/patverga/torch-ner-nlp-from-scratch/raw/master/data/conll2003/eng.train\"\n",
+    "    urllib.request.urlretrieve(url, 'eng.train')\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
    "metadata": {
     "collapsed": true
    },
@@ -47,7 +65,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 4,
    "metadata": {
     "collapsed": false
    },
@@ -104,7 +122,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 5,
    "metadata": {
     "collapsed": false
    },
@@ -155,7 +173,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 6,
    "metadata": {
     "collapsed": false,
     "scrolled": false
@@ -178,7 +196,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 7,
    "metadata": {
     "collapsed": false
    },
@@ -223,7 +241,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 8,
    "metadata": {
     "collapsed": false
    },
@@ -235,7 +253,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 9,
    "metadata": {
     "collapsed": true,
     "scrolled": false

diff --git a/src/main/scala/com/johnsnowlabs/nlp/util/io/ResourceHelper.scala b/src/main/scala/com/johnsnowlabs/nlp/util/io/ResourceHelper.scala
@@ -36,7 +36,7 @@ object ResourceHelper {
 
     if (dirURL != null && dirURL.getProtocol.equals("file")) {
       /* A file path: easy enough */
-      return new File(dirURL.toURI).list()
+      return new File(dirURL.toURI).list().sorted
     }
 
     if (dirURL.getProtocol().equals("jar")) {
@@ -60,7 +60,7 @@ object ResourceHelper {
             result.append(entry)
         }
       }
-      return result.distinct
+      return result.distinct.sorted
     }
 
     throw new UnsupportedOperationException(s"Cannot list files for URL $dirURL")
@@ -69,13 +69,9 @@ object ResourceHelper {
   /** Structure for a SourceStream coming from compiled content */
   case class SourceStream(resource: String) {
     val pipe: Option[InputStream] = try {
-      val stream = if (getClass.getResourceAsStream(resource) != null) {
-        getClass.getResourceAsStream(resource).close()
-        getClass.getResourceAsStream(resource)
-      } else {
-        getClass.getClassLoader.getResourceAsStream(resource).close()
-        getClass.getClassLoader.getResourceAsStream(resource)
-      }
+      var stream = getClass.getResourceAsStream(resource)
+      if (stream == null)
+        stream = getClass.getClassLoader.getResourceAsStream(resource)
 
       Some(stream)
     } catch {