From 35ca9ad0ae93a3c5afabc4480ca492c3ba6caa63 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Wed, 6 Apr 2016 00:54:34 +0200 Subject: [PATCH 1/2] zlib: Make the finish flush flag configurable Up to now, `Z_FINISH` was always the flushing flag that was used for the last chunk of input data. This patch makes this choice configurable so that advanced users can perform e.g. decompression of partial data using `Z_SYNC_FLUSH`, if that suits their needs. Add tests to make sure that an error is thrown upon encountering invalid `flush` or `finishFlush` flags. Fixes: https://github.com/nodejs/node/issues/5761 --- doc/api/zlib.markdown | 1 + lib/zlib.js | 32 ++++++++++++++++---------- test/parallel/test-zlib-flush-flags.js | 28 ++++++++++++++++++++++ test/parallel/test-zlib-truncated.js | 17 ++++++++++++++ 4 files changed, 66 insertions(+), 12 deletions(-) create mode 100644 test/parallel/test-zlib-flush-flags.js diff --git a/doc/api/zlib.markdown b/doc/api/zlib.markdown index da20024ca2109f..49a7d8a429a144 100644 --- a/doc/api/zlib.markdown +++ b/doc/api/zlib.markdown @@ -231,6 +231,7 @@ Note that some options are only relevant when compressing, and are ignored by the decompression classes. * flush (default: `zlib.Z_NO_FLUSH`) +* finishFlush (default: `zlib.Z_FINISH`) * chunkSize (default: 16*1024) * windowBits * level (compression only) diff --git a/lib/zlib.js b/lib/zlib.js index 79c78ea4a51307..a786a0c9118a56 100644 --- a/lib/zlib.js +++ b/lib/zlib.js @@ -234,7 +234,7 @@ function zlibBufferSync(engine, buffer) { if (!(buffer instanceof Buffer)) throw new TypeError('Not a string or buffer'); - var flushFlag = binding.Z_FINISH; + var flushFlag = engine._finishFlushFlag; return engine._processChunk(buffer, flushFlag); } @@ -282,6 +282,14 @@ function Unzip(opts) { Zlib.call(this, opts, binding.UNZIP); } +function isValidFlushFlag(flag) { + return flag === binding.Z_NO_FLUSH || + flag === binding.Z_PARTIAL_FLUSH || + flag === binding.Z_SYNC_FLUSH || + flag === binding.Z_FULL_FLUSH || + flag === binding.Z_FINISH || + flag === binding.Z_BLOCK; +} // the Zlib class they all inherit from // This thing manages the queue of requests, and returns @@ -294,17 +302,16 @@ function Zlib(opts, mode) { Transform.call(this, opts); - if (opts.flush) { - if (opts.flush !== binding.Z_NO_FLUSH && - opts.flush !== binding.Z_PARTIAL_FLUSH && - opts.flush !== binding.Z_SYNC_FLUSH && - opts.flush !== binding.Z_FULL_FLUSH && - opts.flush !== binding.Z_FINISH && - opts.flush !== binding.Z_BLOCK) { - throw new Error('Invalid flush flag: ' + opts.flush); - } + if (opts.flush && !isValidFlushFlag(opts.flush)) { + throw new Error('Invalid flush flag: ' + opts.flush); } + if (opts.finishFlush && !isValidFlushFlag(opts.finishFlush)) { + throw new Error('Invalid flush flag: ' + opts.finishFlush); + } + this._flushFlag = opts.flush || binding.Z_NO_FLUSH; + this._finishFlushFlag = typeof opts.finishFlush !== 'undefined' ? + opts.finishFlush : binding.Z_FINISH; if (opts.chunkSize) { if (opts.chunkSize < exports.Z_MIN_CHUNK || @@ -486,12 +493,13 @@ Zlib.prototype._transform = function(chunk, encoding, cb) { if (this._closed) return cb(new Error('zlib binding closed')); - // If it's the last chunk, or a final flush, we use the Z_FINISH flush flag. + // If it's the last chunk, or a final flush, we use the Z_FINISH flush flag + // (or whatever flag was provided using opts.finishFlush). // If it's explicitly flushing at some other time, then we use // Z_FULL_FLUSH. Otherwise, use Z_NO_FLUSH for maximum compression // goodness. if (last) - flushFlag = binding.Z_FINISH; + flushFlag = this._finishFlushFlag; else { flushFlag = this._flushFlag; // once we've flushed the last of the queue, stop flushing and diff --git a/test/parallel/test-zlib-flush-flags.js b/test/parallel/test-zlib-flush-flags.js new file mode 100644 index 00000000000000..08c79f138b5a58 --- /dev/null +++ b/test/parallel/test-zlib-flush-flags.js @@ -0,0 +1,28 @@ +'use strict'; +require('../common'); +const assert = require('assert'); +const zlib = require('zlib'); + +assert.doesNotThrow(() => { + zlib.createGzip({ flush: zlib.Z_SYNC_FLUSH }); +}); + +assert.throws(() => { + zlib.createGzip({ flush: 'foobar' }); +}, /Invalid flush flag: foobar/); + +assert.throws(() => { + zlib.createGzip({ flush: 10000 }); +}, /Invalid flush flag: 10000/); + +assert.doesNotThrow(() => { + zlib.createGzip({ finishFlush: zlib.Z_SYNC_FLUSH }); +}); + +assert.throws(() => { + zlib.createGzip({ finishFlush: 'foobar' }); +}, /Invalid flush flag: foobar/); + +assert.throws(() => { + zlib.createGzip({ finishFlush: 10000 }); +}, /Invalid flush flag: 10000/); diff --git a/test/parallel/test-zlib-truncated.js b/test/parallel/test-zlib-truncated.js index 38888b68be7084..1798a2ca974451 100644 --- a/test/parallel/test-zlib-truncated.js +++ b/test/parallel/test-zlib-truncated.js @@ -45,5 +45,22 @@ const inputString = 'ΩΩLorem ipsum dolor sit amet, consectetur adipiscing el' zlib[methods.decomp](truncated, function(err, result) { assert(/unexpected end of file/.test(err.message)); }); + + const syncFlushOpt = { finishFlush: zlib.Z_SYNC_FLUSH }; + + // sync truncated input test, finishFlush = Z_SYNC_FLUSH + assert.doesNotThrow(function() { + const result = zlib[methods.decompSync](truncated, syncFlushOpt) + .toString(); + assert.equal(result, inputString.substr(0, result.length)); + }); + + // async truncated input test, finishFlush = Z_SYNC_FLUSH + zlib[methods.decomp](truncated, syncFlushOpt, function(err, decompressed) { + assert.ifError(err); + + const result = decompressed.toString(); + assert.equal(result, inputString.substr(0, result.length)); + }); }); }); From f85f54c2116f353c74e4f4c402b88f2aa5b6cf8e Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Thu, 7 Apr 2016 21:59:25 +0200 Subject: [PATCH 2/2] doc: Add full example for `finishFlush` to zlib docs --- doc/api/zlib.markdown | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/doc/api/zlib.markdown b/doc/api/zlib.markdown index 49a7d8a429a144..92f3f5e52c468e 100644 --- a/doc/api/zlib.markdown +++ b/doc/api/zlib.markdown @@ -109,6 +109,31 @@ http.createServer((request, response) => { }).listen(1337); ``` +By default, the zlib methods with throw an error when decompressing +truncated data. However, if it is known that the data is incomplete, or +the desire is to inspect only the beginning of a compressed file, it is +possible to suppress the default error handling by changing the flushing +method that is used to compressed the last chunk of input data: + +```js +// This is a truncated version of the buffer from the above examples +const buffer = new Buffer('eJzT0yMA', 'base64'); + +zlib.unzip(buffer, { finishFlush: zlib.Z_SYNC_FLUSH }, (err, buffer) => { + if (!err) { + console.log(buffer.toString()); + } else { + // handle error + } +}); +``` + +This will not change the behavior in other error-throwing situations, e.g. +when the input data has an invalid format. Using this method, it will not be +possible to determine whether the input ended prematurely or lacks the +integrity checks, making it necessary to manually check that the +decompressed result is valid. + ## Memory Usage Tuning