Skip to content

Commit 1183ba4

Browse files
eendeegoindutny
eendeego
authored and
indutny
committed
zlib: support concatenated gzip files
Reviewed-By: Fedor Indutny <[email protected]> PR-URL: nodejs/node-v0.x-archive#6442
1 parent 1bb0aeb commit 1183ba4

5 files changed

+282
-8
lines changed

‎lib/zlib.js

+8-1
Original file line numberDiff line numberDiff line change
@@ -580,14 +580,21 @@ Zlib.prototype._processChunk = function(chunk, flushFlag, cb) {
580580
self._buffer = new Buffer(self._chunkSize);
581581
}
582582

583-
if (availOutAfter === 0) {
583+
if (availOutAfter === 0 || availInAfter > 0) {
584584
// Not actually done. Need to reprocess.
585585
// Also, update the availInBefore to the availInAfter value,
586586
// so that if we have to hit it a third (fourth, etc.) time,
587587
// it'll have the correct byte counts.
588588
inOff += (availInBefore - availInAfter);
589589
availInBefore = availInAfter;
590590

591+
if (availOutAfter !== 0) {
592+
// There is still some data available for reading.
593+
// This is usually a concatenated stream, so, reset and restart.
594+
self.reset();
595+
self._offset = 0;
596+
}
597+
591598
if (!async)
592599
return true;
593600

‎src/node_zlib.cc

+24-7
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,11 @@ enum node_zlib_mode {
6363
UNZIP
6464
};
6565

66+
enum node_zlib_error {
67+
NO_ERROR,
68+
FAILED,
69+
WRITE_PENDING
70+
};
6671

6772
void InitZlib(v8::Handle<v8::Object> target);
6873

@@ -203,7 +208,7 @@ class ZCtx : public AsyncWrap {
203208
if (!async) {
204209
// sync version
205210
Process(work_req);
206-
if (CheckError(ctx))
211+
if (CheckError(ctx) == NO_ERROR)
207212
AfterSync(ctx, args);
208213
return;
209214
}
@@ -287,7 +292,7 @@ class ZCtx : public AsyncWrap {
287292
}
288293

289294

290-
static bool CheckError(ZCtx* ctx) {
295+
static node_zlib_error CheckError(ZCtx* ctx) {
291296
// Acceptable error states depend on the type of zlib stream.
292297
switch (ctx->err_) {
293298
case Z_OK:
@@ -300,14 +305,18 @@ class ZCtx : public AsyncWrap {
300305
ZCtx::Error(ctx, "Missing dictionary");
301306
else
302307
ZCtx::Error(ctx, "Bad dictionary");
303-
return false;
308+
return FAILED;
304309
default:
305310
// something else.
306-
ZCtx::Error(ctx, "Zlib error");
307-
return false;
311+
if (ctx->strm_.total_out == 0) {
312+
ZCtx::Error(ctx, "Zlib error");
313+
return FAILED;
314+
} else {
315+
return WRITE_PENDING;
316+
}
308317
}
309318

310-
return true;
319+
return NO_ERROR;
311320
}
312321

313322

@@ -321,7 +330,8 @@ class ZCtx : public AsyncWrap {
321330
HandleScope handle_scope(env->isolate());
322331
Context::Scope context_scope(env->context());
323332

324-
if (!CheckError(ctx))
333+
node_zlib_error error = CheckError(ctx);
334+
if (error == FAILED)
325335
return;
326336

327337
Local<Integer> avail_out = Integer::New(env->isolate(),
@@ -335,6 +345,11 @@ class ZCtx : public AsyncWrap {
335345
Local<Value> args[2] = { avail_in, avail_out };
336346
ctx->MakeCallback(env->callback_string(), ARRAY_SIZE(args), args);
337347

348+
if (error == WRITE_PENDING) {
349+
ZCtx::Error(ctx, "Zlib error");
350+
return;
351+
}
352+
338353
ctx->Unref();
339354
if (ctx->pending_close_)
340355
ctx->Close();
@@ -539,10 +554,12 @@ class ZCtx : public AsyncWrap {
539554
switch (ctx->mode_) {
540555
case DEFLATE:
541556
case DEFLATERAW:
557+
case GZIP:
542558
ctx->err_ = deflateReset(&ctx->strm_);
543559
break;
544560
case INFLATE:
545561
case INFLATERAW:
562+
case GUNZIP:
546563
ctx->err_ = inflateReset(&ctx->strm_);
547564
break;
548565
default:

‎test/simple/test-zlib-from-multiple-gzip-with-garbage.js

+83
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
// Copyright Joyent, Inc. and other Node contributors.
2+
//
3+
// Permission is hereby granted, free of charge, to any person obtaining a
4+
// copy of this software and associated documentation files (the
5+
// "Software"), to deal in the Software without restriction, including
6+
// without limitation the rights to use, copy, modify, merge, publish,
7+
// distribute, sublicense, and/or sell copies of the Software, and to permit
8+
// persons to whom the Software is furnished to do so, subject to the
9+
// following conditions:
10+
//
11+
// The above copyright notice and this permission notice shall be included
12+
// in all copies or substantial portions of the Software.
13+
//
14+
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15+
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16+
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
17+
// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
18+
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
19+
// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
20+
// USE OR OTHER DEALINGS IN THE SOFTWARE.
21+
22+
// test unzipping a file that was created by concatenating multiple gzip
23+
// streams.
24+
25+
var common = require('../common');
26+
var assert = require('assert');
27+
var zlib = require('zlib');
28+
29+
var util = require('util');
30+
31+
var gzipBuffer = new Buffer(128);
32+
var gzipOffset = 0;
33+
34+
var stream1 = '123\n';
35+
var stream2 = '456\n';
36+
var stream3 = '789\n';
37+
38+
function gzipAppend(data) {
39+
data.copy(gzipBuffer, gzipOffset);
40+
gzipOffset += data.length;
41+
}
42+
43+
function writeGzipStream(text, cb) {
44+
var gzip = zlib.createGzip();
45+
gzip.on('data', gzipAppend);
46+
gzip.write(text, function() {
47+
gzip.flush(function() {
48+
gzip.end(function() {
49+
cb();
50+
});
51+
});
52+
});
53+
}
54+
55+
function writeGarbageStream(text, cb) {
56+
gzipAppend(new Buffer(text));
57+
cb();
58+
}
59+
60+
writeGzipStream(stream1, function() {
61+
writeGzipStream(stream2, function() {
62+
writeGarbageStream(stream3, function() {
63+
var gunzip = zlib.createGunzip();
64+
var gunzippedData = new Buffer(2 * 1024);
65+
var gunzippedOffset = 0;
66+
gunzip.on('data', function (data) {
67+
data.copy(gunzippedData, gunzippedOffset);
68+
gunzippedOffset += data.length;
69+
});
70+
gunzip.on('error', function() {
71+
assert.equal(gunzippedData.toString('utf8', 0, gunzippedOffset),
72+
stream1 + stream2);
73+
});
74+
gunzip.on('end', function() {
75+
assert.fail('end event not expected');
76+
});
77+
78+
gunzip.write(gzipBuffer.slice(0, gzipOffset), 'binary', function() {
79+
gunzip.end();
80+
});
81+
});
82+
});
83+
});

‎test/simple/test-zlib-from-multiple-gzip.js

+74
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
// Copyright Joyent, Inc. and other Node contributors.
2+
//
3+
// Permission is hereby granted, free of charge, to any person obtaining a
4+
// copy of this software and associated documentation files (the
5+
// "Software"), to deal in the Software without restriction, including
6+
// without limitation the rights to use, copy, modify, merge, publish,
7+
// distribute, sublicense, and/or sell copies of the Software, and to permit
8+
// persons to whom the Software is furnished to do so, subject to the
9+
// following conditions:
10+
//
11+
// The above copyright notice and this permission notice shall be included
12+
// in all copies or substantial portions of the Software.
13+
//
14+
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15+
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16+
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
17+
// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
18+
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
19+
// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
20+
// USE OR OTHER DEALINGS IN THE SOFTWARE.
21+
22+
// test unzipping a file that was created by concatenating multiple gzip
23+
// streams.
24+
25+
var common = require('../common');
26+
var assert = require('assert');
27+
var zlib = require('zlib');
28+
29+
var util = require('util');
30+
31+
var gzipBuffer = new Buffer(128);
32+
var gzipOffset = 0;
33+
34+
var stream1 = '123\n';
35+
var stream2 = '456\n';
36+
var stream3 = '789\n';
37+
38+
function gzipAppend(data) {
39+
data.copy(gzipBuffer, gzipOffset);
40+
gzipOffset += data.length;
41+
}
42+
43+
function writeGzipStream(text, cb) {
44+
var gzip = zlib.createGzip();
45+
gzip.on('data', gzipAppend);
46+
gzip.write(text, function() {
47+
gzip.flush(function() {
48+
gzip.end(function() {
49+
cb();
50+
});
51+
});
52+
});
53+
}
54+
55+
writeGzipStream(stream1, function() {
56+
writeGzipStream(stream2, function() {
57+
writeGzipStream(stream3, function() {
58+
var gunzip = zlib.createGunzip();
59+
var gunzippedData = new Buffer(2 * 1024);
60+
var gunzippedOffset = 0;
61+
gunzip.on('data', function (data) {
62+
data.copy(gunzippedData, gunzippedOffset);
63+
gunzippedOffset += data.length;
64+
});
65+
gunzip.on('end', function() {
66+
assert.equal(gunzippedData.toString('utf8', 0, gunzippedOffset), stream1 + stream2 + stream3);
67+
});
68+
69+
gunzip.write(gzipBuffer.slice(0, gzipOffset), 'binary', function() {
70+
gunzip.end();
71+
});
72+
});
73+
});
74+
});

‎test/simple/test-zlib-from-multiple-huge-gzip.js

+93
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
// Copyright Joyent, Inc. and other Node contributors.
2+
//
3+
// Permission is hereby granted, free of charge, to any person obtaining a
4+
// copy of this software and associated documentation files (the
5+
// "Software"), to deal in the Software without restriction, including
6+
// without limitation the rights to use, copy, modify, merge, publish,
7+
// distribute, sublicense, and/or sell copies of the Software, and to permit
8+
// persons to whom the Software is furnished to do so, subject to the
9+
// following conditions:
10+
//
11+
// The above copyright notice and this permission notice shall be included
12+
// in all copies or substantial portions of the Software.
13+
//
14+
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15+
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16+
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
17+
// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
18+
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
19+
// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
20+
// USE OR OTHER DEALINGS IN THE SOFTWARE.
21+
22+
// test unzipping a file that was created by concatenating multiple gzip
23+
// streams.
24+
25+
var common = require('../common');
26+
var assert = require('assert');
27+
var zlib = require('zlib');
28+
29+
var util = require('util');
30+
31+
var HUGE = 64 * 1024;
32+
33+
var originalBuffer = new Buffer(3 * HUGE);
34+
var originalOffset = 0;
35+
36+
var gzipBuffer = new Buffer(3 * HUGE);
37+
var gzipOffset = 0;
38+
39+
function getRandomLetter() {
40+
return (Math.random() * (122 - 97)) + 97;
41+
}
42+
43+
function generateHugeStream() {
44+
var buffer = new Buffer(HUGE);
45+
for (var i = 0; i < HUGE; i++)
46+
buffer.writeUInt8(getRandomLetter(), i);
47+
48+
buffer.copy(originalBuffer, originalOffset);
49+
originalOffset += HUGE;
50+
51+
return buffer;
52+
}
53+
54+
function gzipAppend(data) {
55+
data.copy(gzipBuffer, gzipOffset);
56+
gzipOffset += data.length;
57+
}
58+
59+
function writeGzipStream(text, cb) {
60+
var gzip = zlib.createGzip();
61+
gzip.on('data', gzipAppend);
62+
gzip.write(text, function() {
63+
gzip.flush(function() {
64+
gzip.end(function() {
65+
cb();
66+
});
67+
});
68+
});
69+
}
70+
71+
writeGzipStream(generateHugeStream(), function() {
72+
writeGzipStream(generateHugeStream(), function() {
73+
writeGzipStream(generateHugeStream(), function() {
74+
var gunzip = zlib.createGunzip();
75+
var gunzippedData = new Buffer(3 * HUGE);
76+
var gunzippedOffset = 0;
77+
gunzip.on('data', function (data) {
78+
data.copy(gunzippedData, gunzippedOffset);
79+
gunzippedOffset += data.length;
80+
});
81+
gunzip.on('end', function() {
82+
var gunzippedStr = gunzippedData.toString('utf8', 0, gunzippedOffset);
83+
var originalStr = originalBuffer.toString('utf8', 0, 3 * HUGE);
84+
85+
assert.equal(gunzippedStr, originalStr);
86+
});
87+
88+
gunzip.write(gzipBuffer.slice(0, gzipOffset), 'binary', function() {
89+
gunzip.end();
90+
});
91+
});
92+
});
93+
});

0 commit comments

Comments
 (0)