Commit 68df1db7b03 for nodejs
commit 68df1db7b030718cd62fb89eef51960d4c858970
Author: Guilherme Araújo <arauujogui@gmail.com>
Date: Wed Oct 14 15:50:29 2026 -0300
sqlite: use shared-shape objects for result rows
`all()`, `get()` and `iterate()` built each row with the
`Object::New()` overload that takes names and values. That overload
always returns a dictionary-mode object, so no two rows shared a map
and every property read was a hash lookup.
Build rows from a `DictionaryTemplate` cached on the statement in place
of the column names, invalidated on re-prepare. Rows keep their null
prototype. Statements whose column names a template cannot express
(array indices, duplicates, non-ASCII names, which the template
interns as Latin-1) or with more than 64 columns keep the previous
path.
Add a benchmark that reads every column of each row, since the
existing ones only measure building rows.
Fixes: https://github.com/nodejs/node/issues/65799
Assisted-by: Claude
Signed-off-by: Guilherme Araújo <arauujogui@gmail.com>
PR-URL: https://github.com/nodejs/node/pull/66385
Reviewed-By: Trivikram Kamat <trivikr.dev@gmail.com>
Reviewed-By: Matteo Collina <matteo.collina@gmail.com>
diff --git a/benchmark/sqlite/sqlite-prepare-select-read.js b/benchmark/sqlite/sqlite-prepare-select-read.js
new file mode 100644
index 00000000000..63d8a506838
--- /dev/null
+++ b/benchmark/sqlite/sqlite-prepare-select-read.js
@@ -0,0 +1,44 @@
+'use strict';
+const common = require('../common.js');
+const sqlite = require('node:sqlite');
+const assert = require('assert');
+
+const bench = common.createBenchmark(main, {
+ n: [1e4],
+ rows: [1, 100],
+ method: ['all', 'iterate', 'get'],
+ returnArrays: [0, 1],
+});
+
+function main(conf) {
+ const db = new sqlite.Database(':memory:');
+ db.exec(
+ 'CREATE TABLE foo (text_column TEXT, integer_column INTEGER, real_column REAL, blob_column BLOB)',
+ );
+ const insert = db.prepare('INSERT INTO foo VALUES (?, ?, ?, ?)');
+ for (let i = 0; i < conf.rows; i++) {
+ insert.run(`text ${i}`, i, i / 3, Buffer.from('example blob data'));
+ }
+
+ const stmt = db.prepare(`SELECT * FROM foo LIMIT ${conf.rows}`);
+ stmt.setReturnArrays(conf.returnArrays === 1);
+
+ // Each row is built and then has every column read once.
+ const read = conf.returnArrays ?
+ (row) => row[0].length + row[1] + row[2] + row[3].length :
+ (row) => row.text_column.length + row.integer_column +
+ row.real_column + row.blob_column.length;
+
+ let sum = 0;
+ bench.start();
+ for (let i = 0; i < conf.n; i++) {
+ if (conf.method === 'get') {
+ sum += read(stmt.get());
+ } else {
+ for (const row of stmt[conf.method]()) sum += read(row);
+ }
+ }
+ bench.end(conf.n);
+
+ assert.ok(sum > 0);
+}
diff --git a/src/node_sqlite.cc b/src/node_sqlite.cc
index 4f98c6224c7..b4fbfa88746 100644
--- a/src/node_sqlite.cc
+++ b/src/node_sqlite.cc
@@ -22,6 +22,7 @@
#include <cstring>
#include <limits>
#include <string>
+#include <string_view>
namespace node {
namespace sqlite {
@@ -3812,6 +3813,7 @@ void Statement::Finalize() {
void Statement::InvalidateColumnNameCache() {
cached_column_names_.clear();
+ cached_row_template_.Reset();
cached_column_names_reprepare_count_ = -1;
}
@@ -4033,37 +4035,79 @@ MaybeLocal<Name> Statement::ColumnNameToName(const int column) {
.As<Name>();
}
-// Populates `keys` with cached column names, rebuilding the cache if the
-// statement was re-prepared.
-bool Statement::GetCachedColumnNames(LocalVector<Name>* keys) {
- Isolate* isolate = env()->isolate();
+// Size of the stack array RowToObject passes to the template.
+static constexpr int kMaxRowTemplateColumns = 64;
+
+// Templates read names as Latin-1 and reject indices and duplicates.
+static bool CanUseInRowTemplate(Local<Context> context,
+ Local<Name> key,
+ std::string_view name,
+ const std::vector<std::string_view>& seen) {
+ return simdutf::validate_ascii(name.data(), name.size()) &&
+ std::find(seen.begin(), seen.end(), name) == seen.end() &&
+ key->ToArrayIndex(context).IsEmpty();
+}
+// Fills either cached_row_template_ or cached_column_names_.
+bool Statement::UpdateRowCache() {
const int reprepare_count =
sqlite3_stmt_status(statement_.get(), SQLITE_STMTSTATUS_REPREPARE, false);
- if (reprepare_count != cached_column_names_reprepare_count_) {
- cached_column_names_.clear();
- const int num_cols = sqlite3_column_count(statement_.get());
- if (num_cols == 0) {
- cached_column_names_reprepare_count_ = reprepare_count;
- return true;
- }
+ if (reprepare_count == cached_column_names_reprepare_count_) return true;
+
+ InvalidateColumnNameCache();
+ Isolate* isolate = env()->isolate();
+ Local<Context> context = env()->context();
+ const int num_cols = sqlite3_column_count(statement_.get());
+ LocalVector<Name> keys(isolate);
+ std::vector<std::string_view> names;
+ keys.reserve(num_cols);
+ names.reserve(num_cols);
+ bool use_template = num_cols <= kMaxRowTemplateColumns;
+ for (int i = 0; i < num_cols; ++i) {
+ Local<Name> key;
+ if (!ColumnNameToName(i).ToLocal(&key)) return false;
+ std::string_view name = sqlite3_column_name(statement_.get(), i);
+ use_template =
+ use_template && CanUseInRowTemplate(context, key, name, names);
+ keys.push_back(key);
+ names.push_back(name);
+ }
+
+ if (use_template) {
+ cached_row_template_.Reset(
+ isolate,
+ DictionaryTemplate::New(isolate, {names.data(), names.size()}));
+ } else {
cached_column_names_.reserve(num_cols);
- for (int i = 0; i < num_cols; ++i) {
- Local<Name> key;
- if (!ColumnNameToName(i).ToLocal(&key)) {
- InvalidateColumnNameCache();
- return false;
- }
- cached_column_names_.emplace_back(Global<Name>(isolate, key));
+ for (Local<Name> key : keys) {
+ cached_column_names_.emplace_back(isolate, key);
}
- cached_column_names_reprepare_count_ = reprepare_count;
}
+ cached_column_names_reprepare_count_ = reprepare_count;
+ return true;
+}
- keys->reserve(cached_column_names_.size());
- for (const auto& name : cached_column_names_) {
- keys->emplace_back(name.Get(isolate));
+MaybeLocal<Object> Statement::RowToObject(LocalVector<Value>* values) {
+ DCHECK_EQ(
+ sqlite3_stmt_status(statement_.get(), SQLITE_STMTSTATUS_REPREPARE, false),
+ cached_column_names_reprepare_count_);
+ Isolate* isolate = env()->isolate();
+ if (cached_row_template_.IsEmpty()) {
+ LocalVector<Name> keys(isolate);
+ keys.reserve(cached_column_names_.size());
+ for (const auto& name : cached_column_names_) {
+ keys.emplace_back(name.Get(isolate));
+ }
+ DCHECK_EQ(keys.size(), values->size());
+ return Object::New(
+ isolate, Null(isolate), keys.data(), values->data(), keys.size());
}
- return true;
+ DCHECK_LE(values->size(), kMaxRowTemplateColumns);
+ MaybeLocal<Value> template_values[kMaxRowTemplateColumns];
+ std::copy(values->begin(), values->end(), template_values);
+ return NewDictionaryInstanceNullProto(env()->context(),
+ cached_row_template_.Get(isolate),
+ {template_values, values->size()});
}
MaybeLocal<Value> StatementExecutionHelper::ColumnToValue(Environment* env,
@@ -4108,12 +4152,14 @@ MaybeLocal<Value> StatementExecutionHelper::All(Environment* env,
int num_cols = 0;
LocalVector<Value> rows(isolate);
LocalVector<Value> row_values(isolate);
- LocalVector<Name> row_keys(isolate);
SteppingStatementGuard stepping(db, stmt);
while ((r = sqlite3_step(stmt)) == SQLITE_ROW) {
if (num_cols == 0) {
num_cols = sqlite3_column_count(stmt);
+ if (!return_arrays && !statement->UpdateRowCache()) {
+ return MaybeLocal<Value>();
+ }
}
if (ExtractRowValues(env, stmt, num_cols, use_big_ints, &row_values)
@@ -4126,16 +4172,10 @@ MaybeLocal<Value> StatementExecutionHelper::All(Environment* env,
Array::New(isolate, row_values.data(), row_values.size());
rows.emplace_back(row_array);
} else {
- if (row_keys.size() == 0) {
- // Reuses the statement's internalized column names instead of
- // re-interning them on every call.
- if (!statement->GetCachedColumnNames(&row_keys)) {
- return MaybeLocal<Value>();
- }
+ Local<Object> row_obj;
+ if (!statement->RowToObject(&row_values).ToLocal(&row_obj)) {
+ return MaybeLocal<Value>();
}
- DCHECK_EQ(row_keys.size(), row_values.size());
- Local<Object> row_obj = Object::New(
- isolate, Null(isolate), row_keys.data(), row_values.data(), num_cols);
rows.emplace_back(row_obj);
}
}
@@ -4272,17 +4312,9 @@ MaybeLocal<Value> StatementExecutionHelper::Get(Environment* env,
Local<Value> result;
if (return_arrays) {
result = Array::New(isolate, row_values.data(), row_values.size());
- } else {
- LocalVector<Name> keys(isolate);
- // Reuses the statement's internalized column names instead of
- // re-interning them on every call.
- if (!statement->GetCachedColumnNames(&keys)) {
- return MaybeLocal<Value>();
- }
-
- DCHECK_EQ(keys.size(), row_values.size());
- result = Object::New(
- isolate, Null(isolate), keys.data(), row_values.data(), num_cols);
+ } else if (!statement->UpdateRowCache() ||
+ !statement->RowToObject(&row_values).ToLocal(&result)) {
+ return MaybeLocal<Value>();
}
RESET_AND_CHECK(isolate, db, stmt, needs_reset, MaybeLocal<Value>());
@@ -5083,7 +5115,6 @@ void StatementIterator::Next(const FunctionCallbackInfo<Value>& args) {
int num_cols = sqlite3_column_count(iter->stmt_->statement_.get());
Local<Value> row_value;
- LocalVector<Name> row_keys(isolate);
LocalVector<Value> row_values(isolate);
if (ExtractRowValues(env,
@@ -5097,14 +5128,9 @@ void StatementIterator::Next(const FunctionCallbackInfo<Value>& args) {
if (iter->stmt_->return_arrays_) {
row_value = Array::New(isolate, row_values.data(), row_values.size());
- } else {
- // Use cached internalized column names to avoid repeated V8 string
- // creation and enable hidden class sharing across row objects.
- if (!iter->stmt_->GetCachedColumnNames(&row_keys)) return;
-
- DCHECK_EQ(row_keys.size(), row_values.size());
- row_value = Object::New(
- isolate, Null(isolate), row_keys.data(), row_values.data(), num_cols);
+ } else if (!iter->stmt_->UpdateRowCache() ||
+ !iter->stmt_->RowToObject(&row_values).ToLocal(&row_value)) {
+ return;
}
MaybeLocal<Value> values[] = {Boolean::New(isolate, false), row_value};
diff --git a/src/node_sqlite.h b/src/node_sqlite.h
index 0f7f6084696..784f957acaf 100644
--- a/src/node_sqlite.h
+++ b/src/node_sqlite.h
@@ -397,7 +397,9 @@ class Statement : public BaseObject {
static void Dispose(const v8::FunctionCallbackInfo<v8::Value>& args);
v8::MaybeLocal<v8::Value> ColumnToValue(const int column);
v8::MaybeLocal<v8::Name> ColumnNameToName(const int column);
- bool GetCachedColumnNames(v8::LocalVector<v8::Name>* keys);
+ // Call after sqlite3_step(), which may re-prepare, and before RowToObject.
+ bool UpdateRowCache();
+ v8::MaybeLocal<v8::Object> RowToObject(v8::LocalVector<v8::Value>* values);
void Finalize();
bool IsFinalized();
@@ -418,6 +420,8 @@ class Statement : public BaseObject {
inline int ResetStatement();
std::vector<v8::Global<v8::Name>> cached_column_names_;
int cached_column_names_reprepare_count_ = -1;
+ // Set instead of cached_column_names_ when the names fit a template.
+ v8::Global<v8::DictionaryTemplate> cached_row_template_;
void InvalidateColumnNameCache();
bool BindParams(const v8::FunctionCallbackInfo<v8::Value>& args);
bool BindValue(const v8::Local<v8::Value>& value, const int index);
diff --git a/test/parallel/test-sqlite-statement-row-shape.js b/test/parallel/test-sqlite-statement-row-shape.js
new file mode 100644
index 00000000000..2799aba799f
--- /dev/null
+++ b/test/parallel/test-sqlite-statement-row-shape.js
@@ -0,0 +1,79 @@
+// Flags: --allow-natives-syntax
+'use strict';
+const { skipIfSQLiteMissing } = require('../common');
+skipIfSQLiteMissing();
+const assert = require('node:assert');
+const { Database } = require('node:sqlite');
+const { suite, test } = require('node:test');
+
+function assertSharedShape(rows) {
+ for (const row of rows) {
+ assert.strictEqual(Object.getPrototypeOf(row), null);
+ assert(eval('%HasFastProperties(row)'));
+ assert(eval('%HaveSameMap(row, rows[0])'));
+ }
+}
+
+suite('result row objects', () => {
+ const db = new Database(':memory:');
+ db.exec('CREATE TABLE t (a INTEGER, b TEXT, c REAL)');
+ const insert = db.prepare('INSERT INTO t VALUES (?, ?, ?)');
+ insert.run(1, 'x', 1.5);
+ insert.run(2, 'y', 2.5);
+ insert.run(3, null, null);
+
+ test('all() rows share a fast map', () => {
+ const rows = db.prepare('SELECT * FROM t ORDER BY a').all();
+ assert.deepStrictEqual(rows, [
+ { __proto__: null, a: 1, b: 'x', c: 1.5 },
+ { __proto__: null, a: 2, b: 'y', c: 2.5 },
+ { __proto__: null, a: 3, b: null, c: null },
+ ]);
+ assertSharedShape(rows);
+ });
+
+ test('get() rows share a fast map', () => {
+ const stmt = db.prepare('SELECT * FROM t WHERE a = ?');
+ assertSharedShape([stmt.get(1), stmt.get(2), stmt.get(3)]);
+ });
+
+ test('iterate() rows share a fast map', () => {
+ assertSharedShape(db.prepare('SELECT * FROM t ORDER BY a').iterate().toArray());
+ });
+
+ test('a re-prepared statement picks up the new columns', () => {
+ using db = new Database(':memory:');
+ db.exec('CREATE TABLE s (a INTEGER)');
+ db.exec('INSERT INTO s VALUES (1)');
+ const stmt = db.prepare('SELECT * FROM s');
+ assert.deepStrictEqual(stmt.get(), { __proto__: null, a: 1 });
+ db.exec('ALTER TABLE s ADD COLUMN b INTEGER DEFAULT 2');
+ assert.deepStrictEqual(stmt.get(), { __proto__: null, a: 1, b: 2 });
+ });
+
+ test('array-index column names', () => {
+ const stmt = db.prepare('SELECT 1 AS "0", 2 AS b');
+ assert.deepStrictEqual(stmt.get(), { __proto__: null, 0: 1, b: 2 });
+ assert.deepStrictEqual(stmt.all(), [{ __proto__: null, 0: 1, b: 2 }]);
+ });
+
+ test('duplicate column names keep the last value', () => {
+ const stmt = db.prepare('SELECT 1 AS a, 2 AS a');
+ assert.deepStrictEqual(stmt.get(), { __proto__: null, a: 2 });
+ assert.deepStrictEqual(stmt.iterate().toArray(),
+ [{ __proto__: null, a: 2 }]);
+ });
+
+ test('non-ASCII column names', () => {
+ const row = db.prepare('SELECT 1 AS "café", 2 AS "名前"').get();
+ assert.deepStrictEqual(row, { __proto__: null, café: 1, 名前: 2 });
+ });
+
+ test('rows wider than the template limit', () => {
+ const cols = Array.from({ length: 100 }, (_, i) => `${i} AS c${i}`);
+ const row = db.prepare(`SELECT ${cols.join(', ')}`).get();
+ const expected = { __proto__: null };
+ for (let i = 0; i < 100; i++) expected[`c${i}`] = i;
+ assert.deepStrictEqual(row, expected);
+ });
+});