Skip to content

Commit

Permalink
New: add callback style APIs (#4)
Browse files Browse the repository at this point in the history
  • Loading branch information
DonutEspresso authored Oct 27, 2017
1 parent 7f3cff5 commit a12598f
Show file tree
Hide file tree
Showing 3 changed files with 139 additions and 3 deletions.
22 changes: 21 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ continuous flowing data.
This module hopes to fill a gap in the ecosystem: parsing large JSON objects
that are just _really_ big objects. With large in-memory objects, it is
possible to run up against the V8 string length limitation, which is currently
(as of 9/2017) 268435440 characters. Thus, if your large object has enough keys
(as of 9/2017) limited to 512MB. Thus, if your large object has enough keys
or values, it is possible to exceed the string length limit when calling
[JSON.stringify](https://github.com/nodejs/node/issues/10738).

Expand Down Expand Up @@ -98,6 +98,26 @@ __Returns__: {Stream} a JSON.parse stream

__Returns__: {Stream} a JSON.stringify stream

### parse(opts, callback)
An async JSON.parse using the same underlying stream implementation, but with
a callback interface.

* `opts` {Object} an options object passed to `createParseStream`
* `opts.body` {Object} the string to be parsed
* `callback` {Function} a callback object

__Returns__: {Object} the parsed object

### stringify(opts, callback)
An async JSON.stringify using the same underlying stream implementation, but
with a callback interface.

* `opts` {Object} an options object passed to `createStringifyStream`
* `opts.body` {Object} the object to be stringified
* `callback` {Function} a callback object

__Returns__: {Object} the stringified object

## Contributing

Ensure that all linting and codestyle tasks are passing. Add unit tests for any
Expand Down
84 changes: 82 additions & 2 deletions lib/index.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
'use strict';

// core modules
const stream = require('stream');

// external modules
const Assembler = require('stream-json/utils/Assembler');
const assert = require('assert-plus');
Expand Down Expand Up @@ -36,7 +39,8 @@ const utf8Stream = require('utf8-stream');
* any JSON parsing or stringification of large objects won't block the CPU.
* @public
* @param {Object} [options] an options object
* @param {Object} [options.multibyte] when true, support multibyte
* @param {Object} [options.multibyte] when true, support multibyte. defaults
* to true.
* @function createParseStream
* @return {Stream}
*/
Expand Down Expand Up @@ -126,7 +130,83 @@ function createStringifyStream(opts) {
}


/**
* stream based JSON.parse. async function signature to abstract over streams.
* @public
* @param {Object} opts options to pass to parse stream
* @param {String} opts.body string to parse
* @param {Function} callback a callback function
* @return {Object} the parsed JSON object
*/
function parse(opts, callback) {
assert.object(opts, 'opts');
assert.string(opts.body, 'opts.body');
assert.func(callback, 'callback');

const writeStream = through2.obj(function(chunk, enc, cb) {
this.push(chunk);
return cb();
});
const parseStream = createParseStream(opts);

parseStream.on('data', function(pojo) {
return callback(null, pojo);
});

parseStream.on('error', function(err) {
return callback(err);
});

writeStream.pipe(parseStream);
writeStream.end(opts.body);
}


/**
* stream based JSON.stringify. async function signature to abstract over
* streams.
* @public
* @param {Object} opts options to pass to stringify stream
* @param {Function} callback a callback function
* @function parse
* @return {Object} the parsed JSON object
*/
function stringify(opts, callback) {
assert.object(opts, 'opts');
assert.func(callback, 'callback');

let done = false;
let stringified = '';
const stringifyStream = createStringifyStream(opts);
const passthroughStream = new stream.PassThrough();

// setup the passthrough stream as a sink
passthroughStream.on('data', function(chunk) {
stringified += chunk;
});

passthroughStream.on('end', function() {
// if we didn't already error and exit
if (done === false) {
return callback(null, stringified);
}
return null;
});

// don't know what errors stringify stream may emit, but pass them back
// up.
stringifyStream.on('error', function(err) {
done = true;
return callback(err);
});

stringifyStream.pipe(passthroughStream);
}


module.exports = {
createParseStream,
createStringifyStream
createStringifyStream,
parse,
stringify
};
36 changes: 36 additions & 0 deletions test/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -262,4 +262,40 @@ describe('big-json', function() {

});
});


describe('async JSON', function() {
it('should stringify async', function(done) {
json.stringify({
body: POJO
}, function(err, stringified) {
assert.ifError(err);
assert.deepEqual(stringified, JSON.stringify(POJO));
return done();
});
});


it('should parse async', function(done) {
json.parse({
body: JSON.stringify(POJO)
}, function(err, pojo) {
assert.ifError(err);
assert.deepEqual(pojo, POJO);
return done();
});
});


it('should return err in parse async', function(done) {
json.parse({
body: fs.readFileSync(
path.join(__dirname, './etc/corrupt.json')
).toString()
}, function(err, pojo) {
assert.ok(err);
return done();
});
});
});
});

0 comments on commit a12598f

Please sign in to comment.