Commit a143f03cfd8 for nodejs
commit a143f03cfd86847a35eccc09c7db486a5e88115a
Author: James M Snell <jasnell@gmail.com>
Date: Wed Oct 7 04:32:46 2026 +0000
stream: check stream/iter byte views by their byteLength alone
To detect that a byte view was resized or detached after being
accepted, stream/iter recorded, for views not known to be of a
fixed-length, non-shared buffer, a snapshot object with the view's
buffer, the buffer's byteLength and detached state, and the view's
byteLength and byteOffset. Telling the two cases apart read every
view's buffer, which for a small typed array that V8 keeps on the heap,
such as one just allocated by a transform, moves its contents to a new
ArrayBuffer.
What stream/iter accounts for a view is its byteLength, and a view whose
bytes are no longer those accounted for has a different byteLength: a
detached view, a view out of bounds of a shrunk buffer, and a
length-tracking view of a resized or grown buffer. Record and check the
byteLength alone, without reading the buffer, for every view. A
fixed-length view of a resizable buffer that is resized while the view
stays in bounds, whose bytes are unchanged, is no longer rejected.
For 16-byte chunks, piping through a transform that copies each chunk
is about 1.6 times faster, and 2.8 times with pipeToSync(); piping a
sync generator of 64-chunk batches about 1.75 times faster; and piping a
sync generator one chunk per batch to a writer with writeSync() about
1.2-1.3 times faster.
Assisted-by: OpenCode
Signed-off-by: James M Snell <jasnell@gmail.com>
PR-URL: https://github.com/nodejs/node/pull/66504
Reviewed-By: Matteo Collina <matteo.collina@gmail.com>
Reviewed-By: Trivikram Kamat <trivikr.dev@gmail.com>
diff --git a/lib/internal/streams/iter/consumers.js b/lib/internal/streams/iter/consumers.js
index 1da129b0067..a278490ca40 100644
--- a/lib/internal/streams/iter/consumers.js
+++ b/lib/internal/streams/iter/consumers.js
@@ -109,19 +109,19 @@ function collectSync(source, limit) {
// Normalize source via fromSync() - accepts strings, ArrayBuffers, protocols, etc.
const normalized = fromSync(source);
const chunks = [];
- const checks = [];
+ const byteLengths = [];
let totalBytes = 0;
for (const batch of normalized) {
for (let i = 0; i < batch.length; i++) {
- totalBytes += recordChunk(chunks, checks, batch[i]);
+ totalBytes += recordChunk(chunks, byteLengths, batch[i]);
if (limit !== undefined && totalBytes > limit) {
throw new ERR_OUT_OF_RANGE('totalBytes', `<= ${limit}`, totalBytes);
}
}
}
- return validateRecordedChunks(chunks, checks);
+ return validateRecordedChunks(chunks, byteLengths);
}
/**
@@ -137,23 +137,23 @@ async function collectAsync(source, signal, limit) {
// Normalize source via from() - accepts strings, ArrayBuffers, protocols, etc.
const normalized = from(source);
const chunks = [];
- const checks = [];
+ const byteLengths = [];
// Fast path: no signal and no limit
if (!signal && limit === undefined) {
for await (const batch of normalized) {
for (let i = 0; i < batch.length; i++) {
- recordChunk(chunks, checks, batch[i]);
+ recordChunk(chunks, byteLengths, batch[i]);
}
}
- return validateRecordedChunks(chunks, checks);
+ return validateRecordedChunks(chunks, byteLengths);
}
- // Slow path: with signal or limit checks
+ // Slow path: with signal or limit byteLengths
let totalBytes = 0;
function recordBatch(batch) {
for (let i = 0; i < batch.length; i++) {
- totalBytes += recordChunk(chunks, checks, batch[i]);
+ totalBytes += recordChunk(chunks, byteLengths, batch[i]);
if (limit !== undefined && totalBytes > limit) {
throw new ERR_OUT_OF_RANGE('totalBytes', `<= ${limit}`, totalBytes);
}
@@ -162,7 +162,7 @@ async function collectAsync(source, signal, limit) {
if (!signal) {
for await (const batch of normalized) recordBatch(batch);
- return validateRecordedChunks(chunks, checks);
+ return validateRecordedChunks(chunks, byteLengths);
}
// Read as for await...of does, stopping once the signal aborts; see
@@ -187,7 +187,7 @@ async function collectAsync(source, signal, limit) {
}
}
await raceSignal(normalized[SymbolAsyncIterator](), signal, state, consume);
- return validateRecordedChunks(chunks, checks);
+ return validateRecordedChunks(chunks, byteLengths);
}
/**
diff --git a/lib/internal/streams/iter/pull.js b/lib/internal/streams/iter/pull.js
index 37c86f9740c..b8dcd03c761 100644
--- a/lib/internal/streams/iter/pull.js
+++ b/lib/internal/streams/iter/pull.js
@@ -1629,21 +1629,19 @@ function pipeToSync(source, ...args) {
}
if (!hasWritevSync) {
// Chunks written one at a time: snapshot them without an object for
- // every chunk when possible.
+ // every chunk.
const fixed = snapshotFixedBatch(batch);
- if (fixed !== null) {
- for (let i = 0; i < batch.length; i++) {
- const chunk = checkFixedBatchChunk(fixed, i);
- const accepted = writer.writeSync(chunk);
- checkFixedBatchChunk(fixed, i);
- if (accepted === false) {
- throw new ERR_OUT_OF_RANGE(
- 'write', 'within byte budget', 'budget exhausted');
- }
- totalBytes += fixed.byteLengths[i];
+ for (let i = 0; i < batch.length; i++) {
+ const chunk = checkFixedBatchChunk(fixed, i);
+ const accepted = writer.writeSync(chunk);
+ checkFixedBatchChunk(fixed, i);
+ if (accepted === false) {
+ throw new ERR_OUT_OF_RANGE(
+ 'write', 'within byte budget', 'budget exhausted');
}
- continue;
+ totalBytes += fixed.byteLengths[i];
}
+ continue;
}
const entry = createBatchEntry(batch);
if (hasWritevSync && batch.length > 1) {
@@ -1771,9 +1769,6 @@ async function pipeTo(source, ...args) {
}
}
- // Write a batch using try-fallback: sync first, async if needed.
- // Returns undefined on sync success, or a Promise when async fallback
- // is required. Callers must check: const p = writeBatch(b); if (p) await p;
// Write a FixedBatch with writeSync(), like the loop at the end of
// writeBatch(), falling back to the async path for the rest of the batch.
function writeFixedBatch(fixed) {
@@ -1863,6 +1858,9 @@ async function pipeTo(source, ...args) {
}
}
+ // Write a batch using try-fallback: sync first, async if needed.
+ // Returns undefined on sync success, or a Promise when async fallback
+ // is required. Callers must check: const p = writeBatch(b); if (p) await p;
function writeBatch(batch) {
// Single chunk, the common case: check the view around writeSync()
// without allocating a batch entry, and create one only to fall back to
@@ -1876,8 +1874,7 @@ async function pipeTo(source, ...args) {
return writeBatchAsyncFallback(createBatchEntry(batch), 0);
}
if (!hasWritev && hasWriteSync) {
- const fixed = snapshotFixedBatch(batch);
- if (fixed !== null) return writeFixedBatch(fixed);
+ return writeFixedBatch(snapshotFixedBatch(batch));
}
const entry = createBatchEntry(batch);
if (hasWritev && batch.length > 1) {
diff --git a/lib/internal/streams/iter/utils.js b/lib/internal/streams/iter/utils.js
index c22497064df..c4d1e24f0f0 100644
--- a/lib/internal/streams/iter/utils.js
+++ b/lib/internal/streams/iter/utils.js
@@ -3,8 +3,6 @@
const {
Array,
ArrayBufferPrototypeGetByteLength,
- ArrayBufferPrototypeGetDetached,
- ArrayBufferPrototypeGetResizable,
ArrayPrototypeSlice,
FunctionPrototypeCall,
ObjectDefineProperty,
@@ -467,32 +465,20 @@ function toUint8Array(chunk) {
return chunk;
}
-// Byte view snapshots and batch entries are created for every chunk and
-// every batch, so like IterResult they are constructed rather than created
-// as `{ __proto__: null, ... }` literals (which are dictionary-mode objects),
+// Byte views are accepted with their byteLength, and rejected when used if
+// it has changed: if they were detached (byteLength 0), resized with a
+// length-tracking view, or shrunk out of bounds. What was accounted for a
+// view is then what is there. The byteLength is read from the view alone,
+// without its buffer: reading the buffer of a small typed array that V8
+// keeps on the heap moves its contents to a new ArrayBuffer.
+//
+// Byte views and batch entries are created for every chunk and every batch,
+// so like IterResult they are constructed rather than created as
+// `{ __proto__: null, ... }` literals (which are dictionary-mode objects),
// and their prototype is an empty null-prototype object.
-function ByteViewSnapshot(value, buffer, sharedBufferView) {
- this.value = value;
- this.buffer = buffer;
- this.bufferByteLength = sharedBufferView === undefined ?
- ArrayBufferPrototypeGetByteLength(buffer) :
- TypedArrayPrototypeGetByteLength(sharedBufferView);
- this.byteLength = TypedArrayPrototypeGetByteLength(value);
- this.byteOffset = TypedArrayPrototypeGetByteOffset(value);
- this.detached = sharedBufferView === undefined &&
- ArrayBufferPrototypeGetDetached(buffer);
- this.sharedBufferView = sharedBufferView;
-}
-ByteViewSnapshot.prototype = ObjectFreeze({ __proto__: null });
-
-// The snapshot of a non-empty view of a fixed-length, non-shared ArrayBuffer,
-// the common case. Such a view can only change by the buffer being detached,
-// which makes its byteLength 0, so its byteLength is all that needs to be
-// recorded. `buffer` is null to tell it from a ByteViewSnapshot.
function FixedByteView(value, byteLength) {
this.value = value;
this.byteLength = byteLength;
- this.buffer = null;
}
FixedByteView.prototype = ObjectFreeze({ __proto__: null });
@@ -518,48 +504,18 @@ function PendingWrite(batch, resolve, reject) {
PendingWrite.prototype = ObjectFreeze({ __proto__: null });
function snapshotByteView(value) {
- const buffer = TypedArrayPrototypeGetBuffer(value);
- if (isSharedArrayBuffer(buffer)) {
- return new ByteViewSnapshot(value, buffer, new Uint8Array(buffer));
- }
- const byteLength = TypedArrayPrototypeGetByteLength(value);
- if (byteLength !== 0 && !ArrayBufferPrototypeGetResizable(buffer)) {
- return new FixedByteView(value, byteLength);
- }
- return new ByteViewSnapshot(value, buffer, undefined);
+ return new FixedByteView(value, TypedArrayPrototypeGetByteLength(value));
+}
+
+function throwByteViewChanged() {
+ throw new ERR_INVALID_STATE.TypeError(
+ 'Byte view was resized or detached after being accepted');
}
function validateByteView(snapshot) {
- if (snapshot.buffer === null) {
- const { value } = snapshot;
- if (TypedArrayPrototypeGetByteLength(value) !== snapshot.byteLength) {
- throw new ERR_INVALID_STATE.TypeError(
- 'Byte view was resized or detached after being accepted');
- }
- return value;
- }
- const {
- value,
- buffer,
- bufferByteLength,
- byteLength,
- byteOffset,
- detached,
- sharedBufferView,
- } = snapshot;
- const currentBufferByteLength = sharedBufferView === undefined ?
- ArrayBufferPrototypeGetByteLength(buffer) :
- TypedArrayPrototypeGetByteLength(sharedBufferView);
- const currentDetached = sharedBufferView === undefined &&
- ArrayBufferPrototypeGetDetached(buffer);
-
- if (TypedArrayPrototypeGetBuffer(value) !== buffer ||
- currentBufferByteLength !== bufferByteLength ||
- TypedArrayPrototypeGetByteLength(value) !== byteLength ||
- TypedArrayPrototypeGetByteOffset(value) !== byteOffset ||
- currentDetached !== detached) {
- throw new ERR_INVALID_STATE.TypeError(
- 'Byte view was resized or detached after being accepted');
+ const { value } = snapshot;
+ if (TypedArrayPrototypeGetByteLength(value) !== snapshot.byteLength) {
+ throwByteViewChanged();
}
return value;
}
@@ -567,48 +523,17 @@ function validateByteView(snapshot) {
/**
* Call `method` on `receiver` with the byte view `value`, and throw if the
* call resized or detached it, as validateByteView() does for a snapshot
- * taken just before the call. Used for single-chunk writes, the common case:
- * for views on an ArrayBuffer the snapshot is kept in locals instead of a
- * ByteViewSnapshot, so that nothing is allocated per chunk.
+ * taken just before the call, without allocating one.
* @param {Uint8Array} value
* @param {Function} method
* @param {object} receiver
* @returns {any} The result of the call.
*/
function callWithByteView(value, method, receiver) {
- const buffer = TypedArrayPrototypeGetBuffer(value);
- if (!isSharedArrayBuffer(buffer) && !ArrayBufferPrototypeGetResizable(buffer)) {
- // A non-empty view of a fixed-length, non-shared ArrayBuffer can only
- // change by the buffer being detached, which makes its byteLength 0 (see
- // FixedByteView), so checking its byteLength is enough.
- const byteLength = TypedArrayPrototypeGetByteLength(value);
- if (byteLength !== 0) {
- const result = FunctionPrototypeCall(method, receiver, value);
- if (TypedArrayPrototypeGetByteLength(value) !== byteLength) {
- throw new ERR_INVALID_STATE.TypeError(
- 'Byte view was resized or detached after being accepted');
- }
- return result;
- }
- }
- if (isSharedArrayBuffer(buffer)) {
- const snapshot = snapshotByteView(value);
- const result = FunctionPrototypeCall(method, receiver, value);
- validateByteView(snapshot);
- return result;
- }
- const bufferByteLength = ArrayBufferPrototypeGetByteLength(buffer);
const byteLength = TypedArrayPrototypeGetByteLength(value);
- const byteOffset = TypedArrayPrototypeGetByteOffset(value);
- const detached = ArrayBufferPrototypeGetDetached(buffer);
const result = FunctionPrototypeCall(method, receiver, value);
- if (TypedArrayPrototypeGetBuffer(value) !== buffer ||
- ArrayBufferPrototypeGetByteLength(buffer) !== bufferByteLength ||
- TypedArrayPrototypeGetByteLength(value) !== byteLength ||
- TypedArrayPrototypeGetByteOffset(value) !== byteOffset ||
- ArrayBufferPrototypeGetDetached(buffer) !== detached) {
- throw new ERR_INVALID_STATE.TypeError(
- 'Byte view was resized or detached after being accepted');
+ if (TypedArrayPrototypeGetByteLength(value) !== byteLength) {
+ throwByteViewChanged();
}
return result;
}
@@ -690,45 +615,31 @@ function validateBudget(budget) {
}
/**
- * Append a chunk to `chunks` for later concatenation, and the information
- * needed to detect that it was resized or detached in the meantime to
- * `checks`. This provides the same guarantee as snapshotByteView() and
- * validateByteView() without allocating a snapshot for every chunk in the
- * common case: a non-empty view of a fixed-length, non-shared ArrayBuffer can
- * only change by the buffer being detached, which makes its byteLength 0, so
- * its byteLength is all that needs to be recorded.
+ * Append a chunk to `chunks` for later concatenation, and its byteLength to
+ * `byteLengths`, to detect that it was resized or detached in the meantime
+ * as validateByteView() does, without an object for every chunk.
* @param {Uint8Array[]} chunks
- * @param {Array<number|object>} checks
+ * @param {number[]} byteLengths
* @param {Uint8Array} value
* @returns {number} The byteLength of `value`.
*/
-function recordChunk(chunks, checks, value) {
- const buffer = TypedArrayPrototypeGetBuffer(value);
+function recordChunk(chunks, byteLengths, value) {
const byteLength = TypedArrayPrototypeGetByteLength(value);
chunks[chunks.length] = value;
- if (byteLength === 0 || isSharedArrayBuffer(buffer) ||
- ArrayBufferPrototypeGetResizable(buffer)) {
- checks[checks.length] = snapshotByteView(value);
- } else {
- checks[checks.length] = byteLength;
- }
+ byteLengths[byteLengths.length] = byteLength;
return byteLength;
}
/**
* Validate chunks recorded with recordChunk().
* @param {Uint8Array[]} chunks
- * @param {Array<number|object>} checks
+ * @param {number[]} byteLengths
* @returns {Uint8Array[]} `chunks`
*/
-function validateRecordedChunks(chunks, checks) {
+function validateRecordedChunks(chunks, byteLengths) {
for (let i = 0; i < chunks.length; i++) {
- const check = checks[i];
- if (typeof check !== 'number') {
- validateByteView(check);
- } else if (TypedArrayPrototypeGetByteLength(chunks[i]) !== check) {
- throw new ERR_INVALID_STATE.TypeError(
- 'Byte view was resized or detached after being accepted');
+ if (TypedArrayPrototypeGetByteLength(chunks[i]) !== byteLengths[i]) {
+ throwByteViewChanged();
}
}
return chunks;
@@ -745,32 +656,17 @@ FixedBatch.prototype = ObjectFreeze({ __proto__: null });
/**
* Snapshot a batch of Uint8Arrays like createBatchEntry(), without an object
- * for every chunk, when every chunk is a non-empty view of a fixed-length,
- * non-shared ArrayBuffer: such views can only change by being detached,
- * which makes their byteLength 0 (see FixedByteView), so their byteLengths
- * are all that needs to be recorded. Check a chunk with checkFixedBatchChunk()
- * before and after using it.
+ * for every chunk: their byteLengths are recorded in an array. Check a chunk
+ * with checkFixedBatchChunk() before and after using it.
* @param {Uint8Array[]} chunks
- * @returns {FixedBatch|null} null if a chunk needs a full snapshot.
+ * @returns {FixedBatch}
*/
function snapshotFixedBatch(chunks) {
const count = chunks.length;
const byteLengths = new Array(count);
let byteLength = 0;
- let checkedBuffer;
for (let i = 0; i < count; i++) {
- const view = chunks[i];
- const buffer = TypedArrayPrototypeGetBuffer(view);
- // Chunks often share a buffer: check each buffer once.
- if (buffer !== checkedBuffer) {
- if (isSharedArrayBuffer(buffer) ||
- ArrayBufferPrototypeGetResizable(buffer)) {
- return null;
- }
- checkedBuffer = buffer;
- }
- const length = TypedArrayPrototypeGetByteLength(view);
- if (length === 0) return null;
+ const length = TypedArrayPrototypeGetByteLength(chunks[i]);
byteLengths[i] = length;
byteLength += length;
}
@@ -780,8 +676,7 @@ function snapshotFixedBatch(chunks) {
function checkFixedBatchChunk(batch, index) {
const chunk = batch.chunks[index];
if (TypedArrayPrototypeGetByteLength(chunk) !== batch.byteLengths[index]) {
- throw new ERR_INVALID_STATE.TypeError(
- 'Byte view was resized or detached after being accepted');
+ throwByteViewChanged();
}
return chunk;
}
@@ -1082,6 +977,7 @@ module.exports = {
validateBatchEntry,
validateBudget,
validateRecordedChunks,
+ throwByteViewChanged,
validateByteView,
yieldAbortable,
};
diff --git a/test/parallel/test-stream-iter-resizable-buffers.js b/test/parallel/test-stream-iter-resizable-buffers.js
index 1691893d347..3704e73e487 100644
--- a/test/parallel/test-stream-iter-resizable-buffers.js
+++ b/test/parallel/test-stream-iter-resizable-buffers.js
@@ -286,7 +286,28 @@ async function testConsumersRejectDetachedViews() {
assert.strictEqual(result, chunk);
}
+// Only what was accounted for a view is checked: its byteLength. A
+// fixed-length view of a resizable buffer that stays in bounds is unchanged.
+async function testFixedLengthViewOfResizedBuffer() {
+ const buffer = new ArrayBuffer(4, { maxByteLength: 8 });
+ const view = new Uint8Array(buffer, 0, 2);
+ const { writer, readable } = push();
+ assert.strictEqual(writer.writeSync(view), true);
+ buffer.resize(8);
+ writer.endSync();
+ const [chunk] = await array(readable);
+ assert.strictEqual(chunk, view);
+
+ // Shrinking the buffer so that the view is out of bounds makes its
+ // byteLength 0: rejected.
+ const { writer: writer2, readable: readable2 } = push();
+ assert.strictEqual(writer2.writeSync(view), true);
+ buffer.resize(1);
+ await assert.rejects(readable2[Symbol.asyncIterator]().next(), kResizeError);
+}
+
Promise.all([
+ testFixedLengthViewOfResizedBuffer(),
testBufferedViewMutationRejected(),
testDropOldestUsesAcceptedByteLength(),
testPendingWritesRejectResizedViews(),