diff --git a/doc/api/zlib.md b/doc/api/zlib.md index 44caec51b3eb..7093c21abefe 100644 --- a/doc/api/zlib.md +++ b/doc/api/zlib.md @@ -1525,8 +1525,7 @@ which streaming never makes. Output is still bounded per chunk to the declared uncompressed size; pass `maxSize` to impose an explicit ceiling. For an in-memory entry stored without compression, the yielded chunks are -zero-copy views of the entry's retained content (see -[`zipEntry.rawContent`][]); do not mutate them. +zero-copy views of the entry's retained content; do not mutate them. The yielded chunks are **provisional until the iterator completes**. CRC-32 verification (and the final declared-size check) can only run once every byte @@ -1669,20 +1668,56 @@ The entry's raw name bytes, before any character decoding. Useful when the archive's names are in an encoding other than UTF-8 or CP437 and the caller wants to decode them itself. -### `zipEntry.rawContent` +### `zipEntry.rawContent([options])` -* Type: {Buffer|null} +* `options` {Object} + * `maxSize` {number} Reject content with more than this many compressed + bytes, before allocating anything. **Default:** + [`zlib.getMaxZipContentSize()`][]. +* Returns: {Promise} Fulfilled with a {Buffer} containing the entry's raw + content: the bytes as stored in the archive, still compressed if + [`zipEntry.compressed`][] is `true`. The buffer is a fresh copy that shares + no memory with the archive or with data the entry was created from. + +The content is neither decompressed nor checked against +[`zipEntry.crc32`][], which covers the uncompressed bytes. + +Throws an [`ERR_ZIP_ENTRY_TOO_LARGE`][] error if the entry's compressed size +exceeds `maxSize`, and an [`ERR_INVALID_STATE`][] error for a streaming entry +([`zlib.ZipEntry.createStream()`][]) whose content is not yet available. + +### `zipEntry.rawContentSync([options])` + + + +* `options` {Object} See [`zipEntry.rawContent()`][]. +* Returns: {Buffer} The entry's raw content. -The entry's raw (still compressed, if applicable) content when it is held in -memory, or `null` when there is no in-memory buffer to expose - for an entry -created with [`zlib.ZipEntry.createStream()`][], or a file-backed entry -returned by [`zipFile.get()`][], whose bytes are read from disk on demand -rather than retained. Use [`zipEntry.content()`][] or -[`zipEntry.contentIterator()`][] to read a file-backed entry. +The synchronous version of [`zipEntry.rawContent()`][]. + +### `zipEntry.rawContentIterator([options])` + + + +* `options` {Object} + * `maxSize` {number} Reject content with more than this many compressed + bytes, before reading anything. **Default:** no limit. +* Returns: {AsyncIterator} of {Buffer} chunks of the entry's raw content. + +Unlike [`zipEntry.rawContent()`][], this does not buffer the whole member in +memory. For a file-backed entry (one returned by [`zipFile.get()`][]) the +bytes are read from disk as the iterator is consumed and nothing is retained; +the entry is valid only while its `ZipFile` is open. For an in-memory entry, +the yielded chunk is a zero-copy view of the entry's retained content; do not +mutate it. ### `zipEntry.size` @@ -3425,15 +3460,17 @@ Create a Zstandard decompression transform. [`zipBuffer.comment`]: #zipbuffercomment [`zipBuffer.toBuffer()`]: #zipbuffertobufferoptions [`zipBuffer.toBufferSync()`]: #zipbuffertobuffersyncoptions +[`zipEntry.compressed`]: #zipentrycompressed [`zipEntry.content()`]: #zipentrycontentoptions [`zipEntry.contentIterator()`]: #zipentrycontentiteratoroptions [`zipEntry.contentSync()`]: #zipentrycontentsyncoptions +[`zipEntry.crc32`]: #zipentrycrc32 [`zipEntry.isDirectory`]: #zipentryisdirectory [`zipEntry.isSymlink`]: #zipentryissymlink [`zipEntry.modified`]: #zipentrymodified [`zipEntry.nameBuffer`]: #zipentrynamebuffer [`zipEntry.name`]: #zipentryname -[`zipEntry.rawContent`]: #zipentryrawcontent +[`zipEntry.rawContent()`]: #zipentryrawcontentoptions [`zipFile.add()`]: #zipfileaddfilename-data-options [`zipFile.addEntry()`]: #zipfileaddentryentry [`zipFile.close()`]: #zipfileclose diff --git a/lib/internal/zip/entry.js b/lib/internal/zip/entry.js index cfe54903a126..64337d3e57b9 100644 --- a/lib/internal/zip/entry.js +++ b/lib/internal/zip/entry.js @@ -222,7 +222,6 @@ class ZipEntry { // Whether the content is stored in compressed form, whatever the method. get compressed() { return this.method !== METHOD_STORE; } - get rawContent() { return this.#content; } get method() { return this.#meta ? this.#meta.method : this.#central.compressionMethod; } @@ -660,6 +659,59 @@ class ZipEntry { }, options); } + #assertRawWithin(maxSize) { + const declared = this.compressedSize; + if (declared > maxSize) { + throw new ERR_ZIP_ENTRY_TOO_LARGE( + `entry ${JSONStringify(this.name)} declares ${declared} compressed ` + + `bytes, exceeding the ${maxSize} byte limit`); + } + } + + /** + * Reads the entry's raw (still-compressed) content into a single `Buffer`, + * without decompressing or CRC-32-verifying it. + * @param {{ maxSize?: number }} [options] + * @returns {Promise} + */ + async rawContent(options) { + this.#assertRawWithin(options?.maxSize ?? getMaxZipContentSize()); + const handle = this.#fd; + if (handle === null) return Buffer.from(this.#inMemoryCompressed()); + if (handle.closing) { + throw new ERR_INVALID_STATE( + 'cannot read a ZipEntry after its backing ZipFile has been closed'); + } + handle.reads++; + try { + return await this.#compressedBytes(); + } finally { + endHandleRead(handle); + } + } + + /** + * The synchronous counterpart of `rawContent()`. + * @param {{ maxSize?: number }} [options] + * @returns {Buffer} + */ + rawContentSync(options) { + this.#assertRawWithin(options?.maxSize ?? getMaxZipContentSize()); + const compressed = this.#compressedBytesSync(); + return this.#fd === null ? Buffer.from(compressed) : compressed; + } + + /** + * Yields the entry's raw (still-compressed) content as a bounded-memory + * async iterator of `Buffer` chunks, without decompressing or verifying it. + * @param {{ maxSize?: number }} [options] + * @returns {AsyncGenerator} + */ + rawContentIterator(options) { + if (options?.maxSize !== undefined) this.#assertRawWithin(options.maxSize); + return this.#rawSource(); + } + // Synchronous serialization: emit the local file header (sec. 4.3.7) // followed by the raw (already-compressed) content bytes. Streaming // (source-backed) entries cannot be serialized this way. diff --git a/test/parallel/test-zlib-zip-raw-content.js b/test/parallel/test-zlib-zip-raw-content.js new file mode 100644 index 000000000000..f40c99198596 --- /dev/null +++ b/test/parallel/test-zlib-zip-raw-content.js @@ -0,0 +1,114 @@ +'use strict'; + +require('../common'); + +const assert = require('node:assert'); +const zlib = require('node:zlib'); +const fs = require('node:fs/promises'); +const path = require('node:path'); +const tmpdir = require('../common/tmpdir'); +const { test } = require('node:test'); + +tmpdir.refresh(); + +async function buildArchive(entries) { + const chunks = []; + for await (const chunk of zlib.createZipArchive(entries)) chunks.push(chunk); + return Buffer.concat(chunks); +} + +async function collect(iterator) { + const chunks = []; + for await (const chunk of iterator) chunks.push(chunk); + return Buffer.concat(chunks); +} + +const text = Buffer.from('raw content '.repeat(500)); + +function decode(entry, raw) { + switch (entry.method) { + case 0: return raw; + case 8: return zlib.inflateRawSync(raw); + case 93: return zlib.zstdDecompressSync(raw); + } + assert.fail(`unexpected method ${entry.method}`); +} + +async function assertRawReads(entry, expected) { + const raw = await entry.rawContent(); + assert.strictEqual(raw.length, entry.compressedSize); + assert.deepStrictEqual(decode(entry, raw), expected); + assert.deepStrictEqual(entry.rawContentSync(), raw); + assert.deepStrictEqual(await collect(entry.rawContentIterator()), raw); +} + +test('in-memory entries expose their raw bytes for every method', async () => { + for (const method of ['deflate', 'store', 'zstd']) { + const entry = await zlib.ZipEntry.create(`${method}.txt`, text, { method }); + assert.strictEqual(entry.compressed, method !== 'store'); + await assertRawReads(entry, text); + } +}); + +test('parsed and file-backed entries expose their raw bytes', async () => { + const archive = await buildArchive([ + await zlib.ZipEntry.create('d.txt', text), + await zlib.ZipEntry.create('s.txt', text, { method: 'store' }), + await zlib.ZipEntry.create('z.txt', text, { method: 'zstd' }), + await zlib.ZipEntry.create('empty.txt', Buffer.alloc(0)), + ]); + for (const entry of zlib.ZipEntry.read(archive)) { + await assertRawReads(entry, entry.name === 'empty.txt' ? Buffer.alloc(0) : text); + } + + const dir = await fs.mkdtemp(path.join(tmpdir.path, 'zlib-zip-raw-')); + const filePath = path.join(dir, 'archive.zip'); + await fs.writeFile(filePath, archive); + const zip = await zlib.ZipFile.open(filePath); + let entry; + try { + for (const name of zip.keys()) { + entry = await zip.get(name); + await assertRawReads(entry, name === 'empty.txt' ? Buffer.alloc(0) : text); + } + } finally { + await zip.close(); + } + await assert.rejects(entry.rawContent(), { code: 'ERR_INVALID_STATE' }); + assert.throws(() => entry.rawContentSync(), { code: 'ERR_INVALID_STATE' }); + await assert.rejects(collect(entry.rawContentIterator()), { code: 'ERR_INVALID_STATE' }); +}); + +test('rawContent() and rawContentSync() return caller-owned copies', async () => { + const data = Buffer.from('stored bytes'); + const entry = await zlib.ZipEntry.create('s.txt', data, { method: 'store' }); + const raw = await entry.rawContent(); + raw.fill(0); + const rawSync = entry.rawContentSync(); + assert.deepStrictEqual(rawSync, data); + rawSync.fill(0); + assert.deepStrictEqual(await entry.content(), data); + assert.deepStrictEqual(data, Buffer.from('stored bytes')); +}); + +test('maxSize bounds the compressed size', async () => { + const entry = await zlib.ZipEntry.create('s.txt', text, { method: 'store' }); + const limit = { maxSize: text.length - 1 }; + await assert.rejects(entry.rawContent(limit), { code: 'ERR_ZIP_ENTRY_TOO_LARGE' }); + assert.throws(() => entry.rawContentSync(limit), { code: 'ERR_ZIP_ENTRY_TOO_LARGE' }); + assert.throws(() => entry.rawContentIterator(limit), { code: 'ERR_ZIP_ENTRY_TOO_LARGE' }); + + const exact = { maxSize: text.length }; + assert.deepStrictEqual(await entry.rawContent(exact), text); + assert.deepStrictEqual(entry.rawContentSync(exact), text); + assert.deepStrictEqual(await collect(entry.rawContentIterator(exact)), text); +}); + +test('a pending streaming entry has no raw content yet', async () => { + async function* source() { yield text; } + const entry = zlib.ZipEntry.createStream('s.txt', source()); + await assert.rejects(entry.rawContent(), { code: 'ERR_INVALID_STATE' }); + assert.throws(() => entry.rawContentSync(), { code: 'ERR_INVALID_STATE' }); + assert.throws(() => entry.rawContentIterator(), { code: 'ERR_INVALID_STATE' }); + await entry[Symbol.asyncDispose](); +}); diff --git a/test/parallel/test-zlib-zip-vfs.js b/test/parallel/test-zlib-zip-vfs.js index dd85e7251b43..e9fae65e719f 100644 --- a/test/parallel/test-zlib-zip-vfs.js +++ b/test/parallel/test-zlib-zip-vfs.js @@ -49,7 +49,7 @@ test('ZipFile.open() reads a VFS-backed archive through an async virtual fd', as // A lazy, file-backed entry: it retains no content and reads straight // from the virtual fd on demand. const a = await zf.get('a.txt'); - assert.strictEqual(a.rawContent, null); + assert.strictEqual((await a.rawContent()).length, a.compressedSize); assert.strictEqual((await a.content()).toString(), 'hello vfs'); assert.strictEqual(await zf.get('a.txt'), a); // Cached handle identity diff --git a/test/parallel/test-zlib-zip-writable.js b/test/parallel/test-zlib-zip-writable.js index 9e036f62de31..5beb4751ad7b 100644 --- a/test/parallel/test-zlib-zip-writable.js +++ b/test/parallel/test-zlib-zip-writable.js @@ -150,6 +150,7 @@ test('ZipFile addEntry() promotes a spent streaming entry into a readable file-b ]); try { const zip = await zlib.ZipFile.open(filePath, { writable: true }); + let mem; try { const payload = 'streamed payload'.repeat(64); async function* source() { @@ -177,13 +178,13 @@ test('ZipFile addEntry() promotes a spent streaming entry into a readable file-b assert.strictEqual(copy.get('s.txt').crc32 >>> 0, streamEntry.crc32 >>> 0); // An in-memory entry added alongside keeps its own buffer (not promoted). - const mem = await zlib.ZipEntry.create('m.txt', Buffer.from('in memory')); + mem = await zlib.ZipEntry.create('m.txt', Buffer.from('in memory')); await zip.addEntry(mem); - assert.deepStrictEqual(mem.rawContent, mem.rawContent); // still a Buffer - assert.notStrictEqual(mem.rawContent, null); } finally { await zip.close(); } + // Still readable once the ZipFile is closed, so it was not promoted. + assert.strictEqual((await mem.content()).toString(), 'in memory'); // The bytes persisted correctly and re-open sees the streamed member. const reread = await zlib.ZipFile.open(filePath); diff --git a/test/parallel/test-zlib-zip.js b/test/parallel/test-zlib-zip.js index 910c17122d8f..a83159a72b83 100644 --- a/test/parallel/test-zlib-zip.js +++ b/test/parallel/test-zlib-zip.js @@ -83,7 +83,7 @@ test('explicit store option is honored even for compressible data', async () => const data = Buffer.from('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'); const entry = await zlib.ZipEntry.create('stored.txt', data, { method: 'store' }); assert.strictEqual(entry.method, 0); - assert.deepStrictEqual(entry.rawContent, data); + assert.deepStrictEqual(await entry.rawContent(), data); }); test('the zstd method compresses and round-trips through an archive', async () => {