New: add support for multibyte chars (#2)

DonutEspresso · Sep 21, 2017 · ec4bb36 · ec4bb36
1 parent 58c55ad
commit ec4bb36
Show file tree

Hide file tree

Showing 4 changed files with 94 additions and 4 deletions.
diff --git a/README.md b/README.md
@@ -84,7 +84,10 @@ compatibility, use the Node.js streams `pipe()` method.
 
 ## API
 
-### createParseStream()
+### createParseStream(opts)
+
+* `opts` {Object} an options object
+* `opts.multibyte` {Boolean} handle multibyte chars, defaults to true
 
 __Returns__: {Stream} a JSON.parse stream
 

diff --git a/lib/index.js b/lib/index.js
@@ -6,6 +6,7 @@ const assert = require('assert-plus');
 const JSONParseStream = require('stream-json/Combo');
 const through2 = require('through2');
 const stringifyStreamFactory = require('json-stream-stringify');
+const utf8Stream = require('utf8-stream');
 
 
 /**
@@ -34,10 +35,20 @@ const stringifyStreamFactory = require('json-stream-stringify');
  * The advantage of this approach is that by also using a streams interface,
  * any JSON parsing or stringification of large objects won't block the CPU.
  * @public
+ * @param {Object} [options] an options object
+ * @param {Object} [options.multibyte] when true, support multibyte
  * @function createParseStream
  * @return {Stream}
  */
-function createParseStream() {
+function createParseStream(options) {
+
+    const opts = Object.assign({
+        multibyte: true
+    }, options);
+
+    assert.optionalObject(opts, 'opts');
+    assert.optionalBool(opts.multibyte, 'opts.multibyte');
+
     const assembler = new Assembler();
     const parseStream = new JSONParseStream({
         packKeys: true,
@@ -63,7 +74,14 @@ function createParseStream() {
         }
 
         source.unpipe(this);
-        this.transformStream = source.pipe(parseStream);
+
+        // pipe the stream, prepend with multibyte if necessary. utf8stream can
+        // never error, so no need to handle errors here.
+        if (opts.multibyte === true) {
+            this.transformStream = source.pipe(utf8Stream()).pipe(parseStream);
+        } else {
+            this.transformStream = source.pipe(parseStream);
+        }
         redirected = true;
     });
 

diff --git a/package.json b/package.json
@@ -37,6 +37,7 @@
     "assert-plus": "^1.0.0",
     "json-stream-stringify": "^1.5.1",
     "stream-json": "^0.5.2",
-    "through2": "^2.0.3"
+    "through2": "^2.0.3",
+    "utf8-stream": "0.0.0"
   }
 }
diff --git a/test/index.js b/test/index.js
@@ -8,6 +8,7 @@ const stream = require('stream');
 // external modules
 const assert = require('chai').assert;
 const isStream = require('is-stream');
+const through2 = require('through2');
 
 
 // local files
@@ -193,5 +194,72 @@ describe('big-json', function() {
 
             readStream.pipe(parseStream);
         });
+
+
+        it('should handle multibyte keys and vals', function(done) {
+            const multiByte = through2.obj(function(chunk, enc, cb) {
+                this.push(chunk);
+                return cb();
+            });
+
+            const parseStream = json.createParseStream({
+                multibyte: true
+            });
+
+            parseStream.on('data', function(pojo) {
+                assert.deepEqual(pojo, {
+                    '遙': '遙遠未來的事件'
+                });
+                return done();
+            });
+
+            multiByte.pipe(parseStream);
+            multiByte.write('{ "');
+            multiByte.write(Buffer([ 0xe9, 0x81 ]));
+            multiByte.write(Buffer([ 0x99 ]));
+            multiByte.write('":"');
+            multiByte.write(Buffer([ 0xe9, 0x81 ]));
+            multiByte.write(Buffer([ 0x99, 0xe9, 0x81, 0xa0, 0xe6 ]));
+            multiByte.write(Buffer([ 0x9c, 0xaa, 0xe4, 0xbe ]));
+            multiByte.write(Buffer([ 0x86, 0xe7, 0x9a, 0x84,
+                                     0xe4, 0xba, 0x8b ]));
+            multiByte.write(Buffer([ 0xe4, 0xbb, 0xb6 ]));
+            multiByte.end('"}');
+        });
+
+
+        it('should not handle multibyte', function(done) {
+            const multiByte = through2.obj(function(chunk, enc, cb) {
+                this.push(chunk);
+                return cb();
+            });
+
+            const parseStream = json.createParseStream({
+                multibyte: false
+            });
+
+            parseStream.on('data', function(pojo) {
+                assert.deepEqual(pojo, {
+                    // jscs:disable
+                    "���": "���遠������的事件"
+                    // jscs:enable
+                });
+                return done();
+            });
+
+            multiByte.pipe(parseStream);
+            multiByte.write('{ "');
+            multiByte.write(Buffer([ 0xe9, 0x81 ]));
+            multiByte.write(Buffer([ 0x99 ]));
+            multiByte.write('":"');
+            multiByte.write(Buffer([ 0xe9, 0x81 ]));
+            multiByte.write(Buffer([ 0x99, 0xe9, 0x81, 0xa0, 0xe6 ]));
+            multiByte.write(Buffer([ 0x9c, 0xaa, 0xe4, 0xbe ]));
+            multiByte.write(Buffer([ 0x86, 0xe7, 0x9a, 0x84,
+                                     0xe4, 0xba, 0x8b ]));
+            multiByte.write(Buffer([ 0xe4, 0xbb, 0xb6 ]));
+            multiByte.end('"}');
+
+        });
     });
 });