Commit 5c9b456c994 for nodejs
commit 5c9b456c994f9e4f0c624e25b76aa812a51a1aa2
Author: Node.js GitHub Bot <github-bot@iojs.org>
Date: Mon Nov 9 19:53:03 2026 -0500
deps: update simdjson to 5.1.1
PR-URL: https://github.com/nodejs/node/pull/67332
Reviewed-By: Antoine du Hamel <duhamelantoine1995@gmail.com>
Reviewed-By: Colin Ihrig <cjihrig@gmail.com>
Reviewed-By: Luigi Pinca <luigipinca@gmail.com>
diff --git a/deps/simdjson/simdjson.cpp b/deps/simdjson/simdjson.cpp
index 08b0de35294..6db84f29557 100644
--- a/deps/simdjson/simdjson.cpp
+++ b/deps/simdjson/simdjson.cpp
@@ -1,4 +1,4 @@
-/* auto-generated on 2026-10-29 18:49:29 -0400. version 5.1.0 Do not edit! */
+/* auto-generated on 2026-10-31 19:14:12 -0400. version 5.1.1 Do not edit! */
/* including simdjson.cpp: */
/* begin file simdjson.cpp */
#define SIMDJSON_SRC_SIMDJSON_CPP
diff --git a/deps/simdjson/simdjson.h b/deps/simdjson/simdjson.h
index 9d79c053f09..04dbe98eb57 100644
--- a/deps/simdjson/simdjson.h
+++ b/deps/simdjson/simdjson.h
@@ -1,4 +1,4 @@
-/* auto-generated on 2026-10-29 18:49:29 -0400. version 5.1.0 Do not edit! */
+/* auto-generated on 2026-10-31 19:14:12 -0400. version 5.1.1 Do not edit! */
/* including simdjson.h: */
/* begin file simdjson.h */
#ifndef SIMDJSON_H
@@ -2715,7 +2715,7 @@ namespace std {
#define SIMDJSON_SIMDJSON_VERSION_H
/** The version of simdjson being used (major.minor.revision) */
-#define SIMDJSON_VERSION "5.1.0"
+#define SIMDJSON_VERSION "5.1.1"
namespace simdjson {
enum {
@@ -2730,7 +2730,7 @@ enum {
/**
* The revision (major.minor.REVISION) of simdjson being used.
*/
- SIMDJSON_VERSION_REVISION = 0
+ SIMDJSON_VERSION_REVISION = 1
};
} // namespace simdjson
@@ -8405,6 +8405,8 @@ inline constexpr bool enable_view<simdjson::simdjson_result<simdjson::dom::objec
/* skipped duplicate #include "simdjson/dom/element.h" */
/* skipped duplicate #include "simdjson/dom/object.h" */
+#include <cstring>
+
namespace simdjson {
/**
@@ -8492,6 +8494,12 @@ protected:
buffer[size++] = c;
}
+ /** Drops the space characters at the end of the buffer, if any. **/
+ simdjson_inline void trim_trailing_spaces() {
+ while (size > 0 && buffer[size - 1] == ' ')
+ size--;
+ }
+
simdjson_inline void append(const char *begin, const char *end) {
const size_t new_size = size + (end - begin);
if (capacity < new_size)
@@ -8501,6 +8509,26 @@ protected:
size = new_size;
}
+ simdjson_inline void append_n(char c, size_t count) {
+ const size_t new_size = size + count;
+ if (capacity < new_size) {
+ grow(new_size < capacity * 2 ? capacity * 2 : new_size);
+ }
+ std::memset(buffer + size, c, count);
+ size = new_size;
+ }
+
+ /** Makes room for n more bytes and returns where they go. **/
+ simdjson_inline char *reserve(size_t n) {
+ if (capacity - size < n) {
+ grow(size + n < capacity * 2 ? capacity * 2 : size + n);
+ }
+ return buffer + size;
+ }
+
+ /** Ends the content at end, which reserve() returned plus at most n. **/
+ simdjson_inline void set_end(const char *end) { size = size_t(end - buffer); }
+
std::string_view str() const { return std::string_view(buffer, size); }
private:
@@ -8540,6 +8568,8 @@ public:
class pretty_formatter : public base_formatter<pretty_formatter> {
public:
+ explicit pretty_formatter(int step = 4) : indent_step(step) {}
+
simdjson_inline void print_newline();
simdjson_inline void print_indents(size_t depth);
@@ -8564,6 +8594,9 @@ template <class formatter = mini_formatter> class string_builder {
public:
/** Construct an initially empty builder, would print the empty string **/
string_builder() = default;
+ /** Construct a builder whose formatter uses the given indentation step **/
+ explicit string_builder(int indent_step) : format(indent_step) {}
+
/** Append an element to the builder (to be printed) **/
inline void append(simdjson::dom::element value);
/** Append an array to the builder (to be printed) **/
@@ -8705,19 +8738,19 @@ template <class T> std::string minify(simdjson_result<T> x) {
* cout << prettify(doc) << endl;
*
*/
-template <class T> std::string prettify(T x) {
- simdjson::internal::string_builder<simdjson::internal::pretty_formatter> sb;
+template <class T> std::string prettify(T x, int indent_step = 4) {
+ simdjson::internal::string_builder<simdjson::internal::pretty_formatter> sb(indent_step);
sb.append(x);
std::string_view answer = sb.str();
return std::string(answer.data(), answer.size());
}
#if SIMDJSON_EXCEPTIONS
-template <class T> std::string prettify(simdjson_result<T> x) {
+template <class T> std::string prettify(simdjson_result<T> x, int indent_step = 4) {
if (x.error()) {
throw simdjson_error(x.error());
}
- return prettify(x.value());
+ return prettify(x.value(), indent_step);
}
#endif
@@ -10058,7 +10091,16 @@ inline bool object::iterator::key_equals_case_insensitive(std::string_view o) co
// See For case-insensitive string comparisons, avoid char-by-char functions
// https://lemire.me/blog/2020/04/30/for-case-insensitive-string-comparisons-avoid-char-by-char-functions/
// Note that it might be worth rolling our own strncasecmp function, with vectorization.
- return (simdjson_strncasecmp(o.data(), key_c_str(), len) == 0);
+ //
+ // strncasecmp stops at a null character, and a JSON key may contain some:
+ // when it does, we resume the comparison right after the null character.
+ const char *key = key_c_str();
+ size_t i = 0;
+ while (simdjson_strncasecmp(o.data() + i, key + i, len - i) == 0) {
+ const void *null_char = std::memchr(key + i, '\0', len - i);
+ if (null_char == nullptr) { return true; }
+ i = size_t(static_cast<const char *>(null_char) - key) + 1;
+ }
}
return false;
}
@@ -11252,8 +11294,15 @@ inline void document_stream::start() noexcept {
#ifdef SIMDJSON_THREADS_ENABLED
if (use_thread && next_batch_start() < len) {
// Kick off the first thread if needed
+ // load_from_stage1_thread() swaps *parser with stage1_thread_parser, which
+ // then parses every other batch: give it the same settings.
+ stage1_thread_parser.set_max_capacity(parser->max_capacity());
+ stage1_thread_parser.number_as_string(parser->number_as_string());
+ error = stage1_thread_parser.allocate(batch_size, parser->max_depth());
+ if (error) { return; }
error = stage1_thread_parser.ensure_capacity(batch_size);
if (error) { return; }
+ stage1_thread_parser.implementation->_number_as_string = parser->number_as_string();
worker->start_thread();
start_stage1_thread();
if (error) { return; }
@@ -11890,6 +11939,777 @@ simdjson_really_inline char *write_uint_jeaiii(char *b, uint64_t n) noexcept {
#endif // SIMDJSON_INTERNAL_JEAIII_H
/* end file simdjson/internal/jeaiii.h */
+/* including simdjson/internal/json_escape.h: #include "simdjson/internal/json_escape.h" */
+/* begin file simdjson/internal/json_escape.h */
+#ifndef SIMDJSON_INTERNAL_JSON_ESCAPE_H
+#define SIMDJSON_INTERNAL_JSON_ESCAPE_H
+
+// Escaping of JSON strings, shared by the builder and the DOM serializer.
+
+/* skipped duplicate #include "simdjson/base.h" */
+#include <array>
+#include <cstdint>
+#include <cstring>
+#include <string_view>
+#ifdef _MSC_VER
+#include <intrin.h>
+#endif
+
+/*
+ * Empirically, we have found that an inlined optimization is important for
+ * performance. The following macros are not ideal. We should find a better
+ * way to inline the code.
+ */
+
+#if defined(__SSE2__) || defined(__x86_64__) || defined(__x86_64) || \
+ (defined(_M_AMD64) || defined(_M_X64) || \
+ (defined(_M_IX86_FP) && _M_IX86_FP == 2))
+#ifndef SIMDJSON_EXPERIMENTAL_HAS_SSE2
+#define SIMDJSON_EXPERIMENTAL_HAS_SSE2 1
+#endif
+#endif
+
+#if defined(__aarch64__) || defined(_M_ARM64)
+#ifndef SIMDJSON_EXPERIMENTAL_HAS_NEON
+#define SIMDJSON_EXPERIMENTAL_HAS_NEON 1
+#endif
+#endif
+#if defined(__loongarch_sx)
+#ifndef SIMDJSON_EXPERIMENTAL_HAS_LSX
+#define SIMDJSON_EXPERIMENTAL_HAS_LSX 1
+#endif
+#endif
+#if defined(__loongarch_asx)
+#ifndef SIMDJSON_EXPERIMENTAL_HAS_LASX
+#define SIMDJSON_EXPERIMENTAL_HAS_LASX 1
+#endif
+#endif
+#if defined(__riscv_v_intrinsic) && __riscv_v_intrinsic >= 11000 && \
+ defined(__riscv_vector)
+#ifndef SIMDJSON_EXPERIMENTAL_HAS_RVV
+#define SIMDJSON_EXPERIMENTAL_HAS_RVV 1
+#endif
+#endif
+#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) && defined(__POWER8_VECTOR__)
+#ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64
+#define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1
+#endif
+#endif
+#if SIMDJSON_EXPERIMENTAL_HAS_NEON
+#include <arm_neon.h>
+#ifdef _MSC_VER
+#include <intrin.h>
+#endif
+#endif
+#if SIMDJSON_EXPERIMENTAL_HAS_SSE2
+#include <emmintrin.h>
+#if defined(__AVX2__)
+#include <immintrin.h>
+#endif
+#ifdef _MSC_VER
+#include <intrin.h>
+#endif
+#endif
+#if SIMDJSON_EXPERIMENTAL_HAS_LSX
+#include <lsxintrin.h>
+#endif
+#if SIMDJSON_EXPERIMENTAL_HAS_LASX
+#include <lasxintrin.h>
+#endif
+#if SIMDJSON_EXPERIMENTAL_HAS_RVV
+#include <riscv_vector.h>
+#endif
+#if SIMDJSON_EXPERIMENTAL_HAS_PPC64
+#include <altivec.h>
+#ifdef bool
+#undef bool
+#endif
+#ifdef vector
+#undef vector
+#endif
+#endif
+
+namespace simdjson {
+namespace internal {
+
+// The index of the lowest set bit of a non-zero value. The escape code runs
+// outside any implementation, so it cannot use their trailing_zeroes.
+simdjson_inline int trailing_zeroes(uint64_t input_num) noexcept {
+#ifdef _MSC_VER
+ unsigned long ret;
+#if defined(_M_X64) || defined(_M_ARM64)
+ _BitScanForward64(&ret, input_num);
+#else
+ if (!_BitScanForward(&ret, (unsigned long)input_num)) {
+ _BitScanForward(&ret, (unsigned long)(input_num >> 32));
+ ret += 32;
+ }
+#endif
+ return int(ret);
+#else
+ return __builtin_ctzll(input_num);
+#endif
+}
+
+// The tables are static members of a class template rather than namespace-scope
+// statics: a static object would be a different object in every translation
+// unit, and inline functions that index it would break the one-definition rule.
+template <typename T = void> struct escape_tables {
+ // 1 for the bytes that must be escaped: '"', '\\' and control characters.
+ static constexpr uint8_t quotable[256] = {
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+ // The escape sequence of each control character. It is const rather than
+ // constexpr: MSVC rejects a constexpr char[32][7] member of a class template
+ // initialized from string literals (C2131).
+ static const char control[32][7];
+};
+
+#if !SIMDJSON_CPLUSPLUS17
+template <typename T> constexpr uint8_t escape_tables<T>::quotable[256];
+#endif
+template <typename T> const char escape_tables<T>::control[32][7] = {
+ "\\u0000", "\\u0001", "\\u0002", "\\u0003", "\\u0004", "\\u0005", "\\u0006",
+ "\\u0007", "\\b", "\\t", "\\n", "\\u000b", "\\f", "\\r",
+ "\\u000e", "\\u000f", "\\u0010", "\\u0011", "\\u0012", "\\u0013", "\\u0014",
+ "\\u0015", "\\u0016", "\\u0017", "\\u0018", "\\u0019", "\\u001a", "\\u001b",
+ "\\u001c", "\\u001d", "\\u001e", "\\u001f"};
+
+// All Unicode characters may be placed within the quotation marks, except for
+// the characters that MUST be escaped: quotation mark, reverse solidus, and the
+// control characters (U+0000 through U+001F). There are two-character sequence
+// escape representations of some popular characters:
+// \", \\, \b, \f, \n, \r, \t.
+simdjson_inline void escape_json_char(char c, char *&out) {
+ if (c == '"') {
+ memcpy(out, "\\\"", 2);
+ out += 2;
+ } else if (c == '\\') {
+ memcpy(out, "\\\\", 2);
+ out += 2;
+ } else {
+ const char *v = escape_tables<>::control[uint8_t(c)];
+ const size_t n = v[1] == 'u' ? 6 : 2;
+ memcpy(out, v, n);
+ out += n;
+ }
+}
+
+/**
+
+A possible SWAR implementation of has_json_escapable_byte. It is not used
+because it is slower than the current implementation. It is kept here for
+reference (to show that we tried it).
+
+inline bool has_json_escapable_byte(uint64_t x) {
+ uint64_t is_ascii = 0x8080808080808080ULL & ~x;
+ uint64_t xor2 = x ^ 0x0202020202020202ULL;
+ uint64_t lt32_or_eq34 = xor2 - 0x2121212121212121ULL;
+ uint64_t sub92 = x ^ 0x5C5C5C5C5C5C5C5CULL;
+ uint64_t eq92 = (sub92 - 0x0101010101010101ULL);
+ return ((lt32_or_eq34 | eq92) & is_ascii) != 0;
+}
+
+**/
+
+/*
+ * Every variant of the escaper lives in its own namespace (escape_scalar,
+ * escape_sse2, escape_avx2, ...), whose content does not depend on compiler
+ * flags. Callers use escape_best, defined at the end of this file.
+ */
+
+// Scalar fallback for finding next quotable character
+SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline size_t
+find_next_json_quotable_character_scalar(const std::string_view view,
+ size_t location) noexcept {
+ for (auto pos = view.begin() + location; pos != view.end(); ++pos) {
+ if (escape_tables<>::quotable[static_cast<uint8_t>(*pos)]) {
+ return pos - view.begin();
+ }
+ }
+ return size_t(view.size());
+}
+
+// Writes the escaped version of input to out, returning the number of bytes
+// written. Finder::find(input, location) returns the position of the next
+// quotable character at or after location, or input.size().
+template <typename Finder>
+inline size_t write_string_escaped_with(const std::string_view input, char *out) {
+ size_t mysize = input.size();
+
+ // Use SIMD position finder directly - it returns mysize if no escape needed
+ size_t location = Finder::find(input, 0);
+ if (location == mysize) {
+ // Fast path: no escaping needed
+ memcpy(out, input.data(), input.size());
+ return input.size();
+ }
+
+ const char *const initout = out;
+ memcpy(out, input.data(), location);
+ out += location;
+ escape_json_char(input[location], out);
+ location += 1;
+ while (location < mysize) {
+ size_t newlocation = Finder::find(input, location);
+ memcpy(out, input.data() + location, newlocation - location);
+ out += newlocation - location;
+ location = newlocation;
+ if (location == mysize) {
+ break;
+ }
+ escape_json_char(input[location], out);
+ location += 1;
+ }
+ return size_t(out - initout);
+}
+
+namespace escape_scalar {
+struct finder {
+ static simdjson_inline size_t find(const std::string_view view,
+ size_t location) noexcept {
+ return find_next_json_quotable_character_scalar(view, location);
+ }
+};
+
+inline size_t write_string_escaped(const std::string_view input, char *out) {
+ return write_string_escaped_with<finder>(input, out);
+}
+} // namespace escape_scalar
+
+// SIMD-accelerated position finding that directly locates the first quotable
+// character, combining detection and position extraction in a single pass to
+// minimize redundant work.
+#if SIMDJSON_EXPERIMENTAL_HAS_LASX
+namespace escape_lasx {
+simdjson_inline size_t
+find_next_json_quotable_character(const std::string_view view,
+ size_t location) noexcept {
+ const size_t len = view.size();
+ const uint8_t *ptr =
+ reinterpret_cast<const uint8_t *>(view.data()) + location;
+ size_t remaining = len - location;
+
+ // SIMD constants for characters requiring escape
+ __m256i v34 = __lasx_xvreplgr2vr_b(34); // '"'
+ __m256i v92 = __lasx_xvreplgr2vr_b(92); // '\\'
+ __m256i v32 = __lasx_xvreplgr2vr_b(32); // control char threshold
+
+ while (remaining >= 32) {
+ __m256i word = __lasx_xvld(ptr, 0);
+
+ // Check for quotable characters: '"', '\\', or control chars (< 32)
+ __m256i needs_escape = __lasx_xvseq_b(word, v34);
+ needs_escape = __lasx_xvor_v(needs_escape, __lasx_xvseq_b(word, v92));
+ needs_escape = __lasx_xvor_v(needs_escape, __lasx_xvslt_bu(word, v32));
+
+ if (!__lasx_xbz_v(needs_escape)) {
+ // Found a quotable character - locate it via the four 64-bit lanes
+ uint64_t lane0 = __lasx_xvpickve2gr_du(needs_escape, 0);
+ uint64_t lane1 = __lasx_xvpickve2gr_du(needs_escape, 1);
+ uint64_t lane2 = __lasx_xvpickve2gr_du(needs_escape, 2);
+ uint64_t lane3 = __lasx_xvpickve2gr_du(needs_escape, 3);
+ size_t offset = ptr - reinterpret_cast<const uint8_t *>(view.data());
+ if (lane0 != 0) {
+ return offset + trailing_zeroes(lane0) / 8;
+ } else if (lane1 != 0) {
+ return offset + 8 + trailing_zeroes(lane1) / 8;
+ } else if (lane2 != 0) {
+ return offset + 16 + trailing_zeroes(lane2) / 8;
+ } else {
+ return offset + 24 + trailing_zeroes(lane3) / 8;
+ }
+ }
+ ptr += 32;
+ remaining -= 32;
+ }
+ size_t current = len - remaining;
+ return find_next_json_quotable_character_scalar(view, current);
+}
+struct finder {
+ static simdjson_inline size_t find(const std::string_view view,
+ size_t location) noexcept {
+ return find_next_json_quotable_character(view, location);
+ }
+};
+
+inline size_t write_string_escaped(const std::string_view input, char *out) {
+ return write_string_escaped_with<finder>(input, out);
+}
+} // namespace escape_lasx
+#endif
+
+#if SIMDJSON_EXPERIMENTAL_HAS_LSX
+namespace escape_lsx {
+simdjson_inline size_t
+find_next_json_quotable_character(const std::string_view view,
+ size_t location) noexcept {
+ const size_t len = view.size();
+ const uint8_t *ptr =
+ reinterpret_cast<const uint8_t *>(view.data()) + location;
+ size_t remaining = len - location;
+
+ //SIMD constants for characters requiring escape
+ __m128i v34 = __lsx_vreplgr2vr_b(34); // '"'
+ __m128i v92 = __lsx_vreplgr2vr_b(92); // '\\'
+ __m128i v32 = __lsx_vreplgr2vr_b(32); // control char threshold
+
+ while (remaining >= 16){
+ __m128i word = __lsx_vld(ptr, 0);
+
+ //Check for the quotable characters: '"', '\\', or control char (<32)
+ __m128i needs_escape = __lsx_vseq_b(word, v34);
+ needs_escape = __lsx_vor_v(needs_escape, __lsx_vseq_b(word, v92));
+ needs_escape = __lsx_vor_v(needs_escape, __lsx_vslt_bu(word, v32));
+
+ if (!__lsx_bz_v(needs_escape)){
+
+ //Found quotable character - extract exact byte position
+ uint64_t lo = __lsx_vpickve2gr_du(needs_escape,0);
+ uint64_t hi = __lsx_vpickve2gr_du(needs_escape,1);
+ size_t offset = ptr - reinterpret_cast<const uint8_t *>(view.data());
+ if ( lo != 0) {
+ return offset + trailing_zeroes(lo) / 8;
+ } else {
+ return offset + 8 + trailing_zeroes(hi) / 8;
+ }
+ }
+ ptr += 16;
+ remaining -= 16;
+ }
+ size_t current = len - remaining;
+ return find_next_json_quotable_character_scalar(view, current);
+}
+struct finder {
+ static simdjson_inline size_t find(const std::string_view view,
+ size_t location) noexcept {
+ return find_next_json_quotable_character(view, location);
+ }
+};
+
+inline size_t write_string_escaped(const std::string_view input, char *out) {
+ return write_string_escaped_with<finder>(input, out);
+}
+} // namespace escape_lsx
+#endif
+
+#if SIMDJSON_EXPERIMENTAL_HAS_RVV
+namespace escape_rvv {
+simdjson_inline size_t
+find_next_json_quotable_character(const std::string_view view,
+ size_t location) noexcept {
+ const size_t len = view.size();
+ const uint8_t *ptr =
+ reinterpret_cast<const uint8_t *>(view.data()) + location;
+ size_t remaining = len - location;
+
+ while (remaining > 0) {
+ size_t vl = __riscv_vsetvl_e8m1(remaining);
+ vuint8m1_t word = __riscv_vle8_v_u8m1(ptr, vl);
+
+ // Check for quotable characters: '"', '\\', or control chars (< 32)
+ vbool8_t needs_escape = __riscv_vmseq(word, (uint8_t)34, vl);
+ needs_escape = __riscv_vmor(needs_escape,
+ __riscv_vmseq(word, (uint8_t)92, vl), vl);
+ needs_escape = __riscv_vmor(needs_escape,
+ __riscv_vmsltu(word, (uint8_t)32, vl), vl);
+
+ long first = __riscv_vfirst(needs_escape, vl);
+ if (first >= 0) {
+ size_t offset = ptr - reinterpret_cast<const uint8_t *>(view.data());
+ return offset + first;
+ }
+ ptr += vl;
+ remaining -= vl;
+ }
+
+ return len;
+}
+struct finder {
+ static simdjson_inline size_t find(const std::string_view view,
+ size_t location) noexcept {
+ return find_next_json_quotable_character(view, location);
+ }
+};
+
+inline size_t write_string_escaped(const std::string_view input, char *out) {
+ return write_string_escaped_with<finder>(input, out);
+}
+} // namespace escape_rvv
+#endif
+
+#if SIMDJSON_EXPERIMENTAL_HAS_PPC64
+namespace escape_ppc64 {
+simdjson_inline size_t
+find_next_json_quotable_character(const std::string_view view,
+ size_t location) noexcept {
+ const size_t len = view.size();
+ const uint8_t *ptr =
+ reinterpret_cast<const uint8_t *>(view.data()) + location;
+ size_t remaining = len - location;
+
+ // SIMD constants for characters requiring escape
+ __vector unsigned char v34 = vec_splats((unsigned char)34); // '"'
+ __vector unsigned char v92 = vec_splats((unsigned char)92); // '\\'
+ __vector unsigned char v32 = vec_splats((unsigned char)32); // control char threshold
+
+ // Bitmask for vec_vbpermq to extract one bit per byte
+ const __vector unsigned char perm_mask = {0x78, 0x70, 0x68, 0x60, 0x58, 0x50,
+ 0x48, 0x40, 0x38, 0x30, 0x28, 0x20,
+ 0x18, 0x10, 0x08, 0x00};
+
+ while (remaining >= 16) {
+ __vector unsigned char word =
+ vec_vsx_ld(0, reinterpret_cast<const unsigned char *>(ptr));
+
+ // Check for quotable characters: '"', '\\', or control chars (< 32)
+ __vector unsigned char needs_escape =
+ (__vector unsigned char)vec_cmpeq(word, v34);
+ needs_escape = vec_or(needs_escape,
+ (__vector unsigned char)vec_cmpeq(word, v92));
+ needs_escape = vec_or(needs_escape,
+ (__vector unsigned char)vec_cmplt(word, v32));
+
+ __vector unsigned long long result =
+ (__vector unsigned long long)vec_vbpermq(needs_escape, perm_mask);
+#ifdef __LITTLE_ENDIAN__
+ unsigned int mask = static_cast<unsigned int>(result[1]);
+#else
+ unsigned int mask = static_cast<unsigned int>(result[0]);
+#endif
+ if (mask != 0) {
+ size_t offset = ptr - reinterpret_cast<const uint8_t *>(view.data());
+ return offset + __builtin_ctz(mask);
+ }
+ ptr += 16;
+ remaining -= 16;
+ }
+
+ // Scalar fallback for remaining bytes
+ size_t current = len - remaining;
+ return find_next_json_quotable_character_scalar(view, current);
+}
+struct finder {
+ static simdjson_inline size_t find(const std::string_view view,
+ size_t location) noexcept {
+ return find_next_json_quotable_character(view, location);
+ }
+};
+
+inline size_t write_string_escaped(const std::string_view input, char *out) {
+ return write_string_escaped_with<finder>(input, out);
+}
+} // namespace escape_ppc64
+#endif
+
+#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 || SIMDJSON_EXPERIMENTAL_HAS_NEON
+// Block escaping: write_string_escaped scans the input itself, 16 bytes at a
+// time. The two instruction sets are never available together.
+#if SIMDJSON_EXPERIMENTAL_HAS_SSE2
+namespace escape_sse2 {
+#else
+namespace escape_neon {
+#endif
+
+#if SIMDJSON_EXPERIMENTAL_HAS_SSE2
+
+using escape_vector = __m128i;
+// Mask bits that each input byte contributes to escape_bitmask().
+static constexpr unsigned escape_mask_bits = 1;
+
+simdjson_inline escape_vector escape_load16(const uint8_t *p) noexcept {
+ return _mm_loadu_si128(reinterpret_cast<const __m128i *>(p));
+}
+
+simdjson_inline void escape_store16(char *out, escape_vector v) noexcept {
+ _mm_storeu_si128(reinterpret_cast<__m128i *>(out), v);
+}
+
+// Builds a vector whose bytes 0..7 come from a and bytes 8..15 from b.
+simdjson_inline escape_vector escape_load8x2(const uint8_t *a,
+ const uint8_t *b) noexcept {
+ return _mm_unpacklo_epi64(
+ _mm_loadl_epi64(reinterpret_cast<const __m128i *>(a)),
+ _mm_loadl_epi64(reinterpret_cast<const __m128i *>(b)));
+}
+
+// Builds a vector whose bytes 0..3 come from a and bytes 4..7 from b. The
+// upper half is unspecified; callers only look at the low eight bits.
+simdjson_inline escape_vector escape_load4x2(const uint8_t *a,
+ const uint8_t *b) noexcept {
+ int32_t a32, b32;
+ memcpy(&a32, a, 4);
+ memcpy(&b32, b, 4);
+ return _mm_unpacklo_epi32(_mm_cvtsi32_si128(a32), _mm_cvtsi32_si128(b32));
+}
+
+// Sets every bit of byte k when byte k of v is a quotable character ('"',
+// '\\' or a control character).
+simdjson_inline escape_vector escape_flags(escape_vector v) noexcept {
+ const __m128i v34 = _mm_set1_epi8(34); // '"'
+ const __m128i v92 = _mm_set1_epi8(92); // '\\'
+ const __m128i v31 = _mm_set1_epi8(31); // for control char detection
+ __m128i needs_escape = _mm_cmpeq_epi8(v, v34);
+ needs_escape = _mm_or_si128(needs_escape, _mm_cmpeq_epi8(v, v92));
+ return _mm_or_si128(
+ needs_escape, _mm_cmpeq_epi8(_mm_subs_epu8(v, v31), _mm_setzero_si128()));
+}
+
+// True when any byte needs escaping. Kept separate from escape_bitmask
+// because some instruction sets can answer it without leaving the vector
+// register file; on SSE2 the compiler folds the two together.
+simdjson_inline bool escape_any(escape_vector flags) noexcept {
+ return _mm_movemask_epi8(flags) != 0;
+}
+
+// A 16-bit mask with bit k set when byte k needed escaping.
+simdjson_inline uint64_t escape_bitmask(escape_vector flags) noexcept {
+ return uint64_t(uint32_t(_mm_movemask_epi8(flags)));
+}
+
+#else // SIMDJSON_EXPERIMENTAL_HAS_NEON
+
+using escape_vector = uint8x16_t;
+static constexpr unsigned escape_mask_bits = 4;
+
+simdjson_inline escape_vector escape_load16(const uint8_t *p) noexcept {
+ return vld1q_u8(p);
+}
+
+simdjson_inline void escape_store16(char *out, escape_vector v) noexcept {
+ vst1q_u8(reinterpret_cast<uint8_t *>(out), v);
+}
+
+simdjson_inline escape_vector escape_load8x2(const uint8_t *a,
+ const uint8_t *b) noexcept {
+ uint64_t a64, b64;
+ memcpy(&a64, a, 8);
+ memcpy(&b64, b, 8);
+ return vreinterpretq_u8_u64(vsetq_lane_u64(b64, vdupq_n_u64(a64), 1));
+}
+
+simdjson_inline escape_vector escape_load4x2(const uint8_t *a,
+ const uint8_t *b) noexcept {
+ uint32_t a32, b32;
+ memcpy(&a32, a, 4);
+ memcpy(&b32, b, 4);
+ return vreinterpretq_u8_u32(vsetq_lane_u32(b32, vdupq_n_u32(a32), 1));
+}
+
+simdjson_inline escape_vector escape_flags(escape_vector v) noexcept {
+ uint8x16_t needs_escape = vceqq_u8(v, vdupq_n_u8(34)); // '"'
+ needs_escape = vorrq_u8(needs_escape, vceqq_u8(v, vdupq_n_u8(92))); // '\\'
+ return vorrq_u8(needs_escape, vcltq_u8(v, vdupq_n_u8(32)));
+}
+
+simdjson_inline bool escape_any(escape_vector flags) noexcept {
+ return vmaxvq_u32(vreinterpretq_u32_u8(flags)) != 0;
+}
+
+// Four bits per byte rather than one: escape_block and the tail paths scale
+// their shifts by escape_mask_bits to match.
+simdjson_inline uint64_t escape_bitmask(escape_vector flags) noexcept {
+ uint8x8_t narrowed = vshrn_n_u16(vreinterpretq_u16_u8(flags), 4);
+ return vget_lane_u64(vreinterpret_u64_u8(narrowed), 0);
+}
+
+#endif // instruction set selection
+// The escape bitmask of a 16-byte block.
+simdjson_inline uint64_t escape_mask(escape_vector v) noexcept {
+ return escape_bitmask(escape_flags(v));
+}
+
+// Copies n bytes with n < 16, using overlapping loads and stores. It never
+// reads more than n bytes from src, nor writes more than n bytes to dst.
+simdjson_inline void copy_lt16(char *dst, const uint8_t *src,
+ size_t n) noexcept {
+ if (n >= 8) {
+ memcpy(dst, src, 8);
+ memcpy(dst + n - 8, src + n - 8, 8);
+ } else if (n >= 4) {
+ memcpy(dst, src, 4);
+ memcpy(dst + n - 4, src + n - 4, 4);
+ } else if (n > 0) {
+ dst[0] = char(src[0]);
+ dst[n >> 1] = char(src[n >> 1]);
+ dst[n - 1] = char(src[n - 1]);
+ }
+}
+
+// Escapes the bytes of src in the range [i, blockend), given that m is the
+// (non-zero) escape mask for that range: the escape_mask_bits-wide lane of m
+// at byte k is non-zero when src[i + k] requires escaping. Returns the updated
+// output pointer.
+simdjson_never_inline char *escape_block(const uint8_t *src, char *out,
+ size_t i, size_t blockend,
+ uint64_t m) noexcept {
+ constexpr uint64_t lane = (uint64_t(1) << escape_mask_bits) - 1;
+
+ size_t pos = i; // first byte not yet copied
+ while (m) {
+ const size_t tz = trailing_zeroes(m);
+ const size_t next = i + tz / escape_mask_bits;
+ // Copy the run of safe bytes that precedes this escape.
+ copy_lt16(out, src + pos, next - pos);
+ out += next - pos;
+ escape_json_char(char(src[next]), out);
+ pos = next + 1;
+ m &= ~(lane << tz);
+ }
+ // Copy whatever follows the last escape.
+ copy_lt16(out, src + pos, blockend - pos);
+ return out + (blockend - pos);
+}
+
+// Escapes src[i, len) to out, returning the updated output pointer. The input
+// must be the whole string: when len >= 16, the tail reloads its last 16 bytes.
+simdjson_really_inline char *escape_from(const uint8_t *src, size_t len,
+ size_t i, char *out) noexcept {
+ while (i + 16 <= len) {
+ escape_vector word = escape_load16(src + i);
+ escape_vector flags = escape_flags(word);
+ if (simdjson_likely(!escape_any(flags))) {
+ escape_store16(out, word);
+ out += 16;
+ } else {
+ out = escape_block(src, out, i, i + 16, escape_bitmask(flags));
+ }
+ i += 16;
+ }
+ if (i < len) {
+ const size_t rem = len - i;
+ uint64_t m;
+ if (len >= 16) {
+ // The last 16 bytes of the input are in bounds. Bit k of that block's
+ // mask belongs to input position len - 16 + k, so shift it down to align
+ // bit 0 with position i.
+ m = escape_mask(escape_load16(src + len - 16)) >>
+ (escape_mask_bits * (16 - rem));
+ } else if (len >= 8) {
+ // Here i == 0 and rem == len. Two overlapping 8-byte loads cover
+ // [0, 8) and [len - 8, len), which is the whole input since len < 16.
+ uint64_t mm = escape_mask(escape_load8x2(src, src + len - 8));
+ constexpr uint64_t low8 = (uint64_t(1) << (escape_mask_bits * 8)) - 1;
+ m = (mm & low8) |
+ ((mm >> (escape_mask_bits * 8)) << (escape_mask_bits * (len - 8)));
+ } else if (len >= 4) {
+ // Same idea with two overlapping 4-byte loads.
+ uint64_t mm = escape_mask(escape_load4x2(src, src + len - 4));
+ constexpr uint64_t low4 = (uint64_t(1) << (escape_mask_bits * 4)) - 1;
+ m = (mm & low4) | (((mm >> (escape_mask_bits * 4)) & low4)
+ << (escape_mask_bits * (len - 4)));
+ } else {
+ // Fewer than 4 bytes: at most three table lookups, no need for SIMD.
+ for (size_t k = 0; k < len; k++) {
+ uint8_t c = src[k];
+ if (escape_tables<>::quotable[c]) {
+ escape_json_char(char(c), out);
+ } else {
+ *out++ = char(c);
+ }
+ }
+ return out;
+ }
+ if (m == 0) {
+ copy_lt16(out, src + i, rem);
+ out += rem;
+ } else {
+ out = escape_block(src, out, i, len, m);
+ }
+ }
+ return out;
+}
+
+// Writes the escaped version of input to out, returning the number of bytes
+// written.
+simdjson_really_inline size_t write_string_escaped(const std::string_view input, char *out) {
+ const uint8_t *src = reinterpret_cast<const uint8_t *>(input.data());
+ return size_t(escape_from(src, input.size(), 0, out) - out);
+}
+
+#if SIMDJSON_EXPERIMENTAL_HAS_SSE2
+} // namespace escape_sse2
+#else
+} // namespace escape_neon
+#endif
+#endif // SIMDJSON_EXPERIMENTAL_HAS_SSE2 || SIMDJSON_EXPERIMENTAL_HAS_NEON
+
+#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 && defined(__AVX2__)
+// SSE2 with a 32-byte main loop. It has its own namespace because it is
+// compiled only when __AVX2__ is defined.
+namespace escape_avx2 {
+simdjson_really_inline size_t write_string_escaped(const std::string_view input, char *out) {
+ using namespace escape_sse2;
+ const size_t len = input.size();
+ const uint8_t *src = reinterpret_cast<const uint8_t *>(input.data());
+ const char *const initout = out;
+
+ size_t i = 0;
+ while (i + 32 <= len) {
+ const __m256i word = _mm256_loadu_si256(reinterpret_cast<const __m256i *>(src + i));
+ const __m256i flags = _mm256_or_si256(
+ _mm256_or_si256(_mm256_cmpeq_epi8(word, _mm256_set1_epi8(34)), // '"'
+ _mm256_cmpeq_epi8(word, _mm256_set1_epi8(92))), // '\\'
+ _mm256_cmpeq_epi8(_mm256_subs_epu8(word, _mm256_set1_epi8(31)),
+ _mm256_setzero_si256())); // control
+ const uint32_t mask = uint32_t(_mm256_movemask_epi8(flags));
+ if (simdjson_likely(mask == 0)) {
+ _mm256_storeu_si256(reinterpret_cast<__m256i *>(out), word);
+ out += 32;
+ } else {
+ for (size_t half = 0; half < 32; half += 16) {
+ const uint64_t m = (mask >> half) & 0xFFFF;
+ if (m == 0) {
+ escape_store16(out, escape_load16(src + i + half));
+ out += 16;
+ } else {
+ out = escape_block(src, out, i + half, i + half + 16, m);
+ }
+ }
+ }
+ i += 32;
+ }
+ return size_t(escape_from(src, len, i, out) - initout);
+}
+} // namespace escape_avx2
+#endif
+
+// The variant that the builder and the DOM serializer use: the best one that
+// the compiler flags allow. Translation units compiled with different flags
+// may select different variants for the same inline functions; see
+// "Mixing compiler flags" in doc/implementation-selection.md.
+#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 && defined(__AVX2__)
+namespace escape_best = escape_avx2;
+#elif SIMDJSON_EXPERIMENTAL_HAS_SSE2
+namespace escape_best = escape_sse2;
+#elif SIMDJSON_EXPERIMENTAL_HAS_NEON
+namespace escape_best = escape_neon;
+#elif SIMDJSON_EXPERIMENTAL_HAS_LASX
+namespace escape_best = escape_lasx;
+#elif SIMDJSON_EXPERIMENTAL_HAS_LSX
+namespace escape_best = escape_lsx;
+#elif SIMDJSON_EXPERIMENTAL_HAS_RVV
+namespace escape_best = escape_rvv;
+#elif SIMDJSON_EXPERIMENTAL_HAS_PPC64
+namespace escape_best = escape_ppc64;
+#else
+namespace escape_best = escape_scalar;
+#endif
+
+} // namespace internal
+} // namespace simdjson
+
+#endif // SIMDJSON_INTERNAL_JSON_ESCAPE_H
+/* end file simdjson/internal/json_escape.h */
/* skipped duplicate #include "simdjson/internal/tape_type.h" */
/* skipped duplicate #include "simdjson/dom/array-inl.h" */
@@ -11962,20 +12782,6 @@ operator<<(std::ostream &out,
} // namespace dom
-/***
- * Number utility functions
- **/
-namespace {
-/**@private
- * Escape sequence like \b or \u0001
- * We expect that most compilers will use 8 bytes for this data structure.
- **/
-struct escape_sequence {
- uint8_t length;
- const char
- string[7]; // technically, we only ever need 6 characters, we pad to 8
-};
-} // anonymous namespace
namespace internal {
/***
@@ -12091,86 +12897,14 @@ template <class formatter>
simdjson_inline void
base_formatter<formatter>::string(std::string_view unescaped) {
one_char('\"');
- size_t i = 0;
- // Fast path for the case where we have no control character, no ", and no
- // backslash. This should include most keys.
- //
- // We would like to use 'bool' but some compilers take offense to bitwise
- // operation with bool types.
- constexpr static char needs_escaping[] = {
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
- for (; i + 8 <= unescaped.length(); i += 8) {
- // Poor's man vectorization. This could get much faster if we used SIMD.
- //
- // It is not the case that replacing '|' with '||' would be neutral
- // performance-wise.
- if (needs_escaping[uint8_t(unescaped[i])] |
- needs_escaping[uint8_t(unescaped[i + 1])] |
- needs_escaping[uint8_t(unescaped[i + 2])] |
- needs_escaping[uint8_t(unescaped[i + 3])] |
- needs_escaping[uint8_t(unescaped[i + 4])] |
- needs_escaping[uint8_t(unescaped[i + 5])] |
- needs_escaping[uint8_t(unescaped[i + 6])] |
- needs_escaping[uint8_t(unescaped[i + 7])]) {
- break;
- }
- }
- for (; i < unescaped.length(); i++) {
- if (needs_escaping[uint8_t(unescaped[i])]) {
- break;
- }
+ // Each input byte becomes at most 6 output bytes. Escaping the string in
+ // pieces bounds the room reserved ahead of a long string.
+ constexpr size_t piece = 4096;
+ for (size_t i = 0; i < unescaped.size(); i += piece) {
+ const std::string_view part = unescaped.substr(i, piece);
+ char *out = buffer.reserve(6 * part.size());
+ buffer.set_end(out + escape_best::write_string_escaped(part, out));
}
- // The following is also possible and omits a 256-byte table, but it is
- // slower: for (; (i < unescaped.length()) && (uint8_t(unescaped[i]) > 0x1F)
- // && (unescaped[i] != '\"') && (unescaped[i] != '\\'); i++) {}
-
- // At least for long strings, the following should be fast. We could
- // do better by integrating the checks and the insertion.
- chars(unescaped.data(), unescaped.data() + i);
- // We caught a control character if we enter this loop (slow).
- // Note that we are do not restart from the beginning, but rather we continue
- // from the point where we encountered something that requires escaping.
- for (; i < unescaped.length(); i++) {
- switch (unescaped[i]) {
- case '\"': {
- const char *s = "\\\"";
- chars(s, s + 2);
- } break;
- case '\\': {
- const char *s = "\\\\";
- chars(s, s + 2);
- } break;
- default:
- if (uint8_t(unescaped[i]) <= 0x1F) {
- // If packed, this uses 8 * 32 bytes.
- // Note that we expect most compilers to embed this code in the data
- // section.
- constexpr static escape_sequence escaped[32] = {
- {6, "\\u0000"}, {6, "\\u0001"}, {6, "\\u0002"}, {6, "\\u0003"},
- {6, "\\u0004"}, {6, "\\u0005"}, {6, "\\u0006"}, {6, "\\u0007"},
- {2, "\\b"}, {2, "\\t"}, {2, "\\n"}, {6, "\\u000b"},
- {2, "\\f"}, {2, "\\r"}, {6, "\\u000e"}, {6, "\\u000f"},
- {6, "\\u0010"}, {6, "\\u0011"}, {6, "\\u0012"}, {6, "\\u0013"},
- {6, "\\u0014"}, {6, "\\u0015"}, {6, "\\u0016"}, {6, "\\u0017"},
- {6, "\\u0018"}, {6, "\\u0019"}, {6, "\\u001a"}, {6, "\\u001b"},
- {6, "\\u001c"}, {6, "\\u001d"}, {6, "\\u001e"}, {6, "\\u001f"}};
- auto u = escaped[uint8_t(unescaped[i])];
- chars(u.string, u.string + u.length);
- } else {
- one_char(unescaped[i]);
- }
- } // switch
- } // for
one_char('\"');
}
@@ -12195,12 +12929,10 @@ simdjson_inline void mini_formatter::print_space() { return; }
simdjson_inline void pretty_formatter::print_newline() { one_char('\n'); }
simdjson_inline void pretty_formatter::print_indents(size_t depth) {
- if (this->indent_step <= 0) {
+ if (indent_step <= 0) {
return;
}
- for (size_t i = 0; i < this->indent_step * depth; i++) {
- one_char(' ');
- }
+ buffer.append_n(' ', depth * static_cast<size_t>(indent_step));
}
simdjson_inline void pretty_formatter::print_space() { one_char(' '); }
@@ -12790,7 +13522,7 @@ private:
/** Row for a uniform scalar array: writes elem inline, then pads to width so every row lines up. */
void format_table_scalar_row(const dom::element& elem, const element_metrics& row_metrics,
- size_t width, size_t depth, table_column_type column_type);
+ size_t width, table_column_type column_type);
/** Shared per-column writer: recurses if the column has children,
* otherwise writes a plain padded value or blank. */
@@ -12803,7 +13535,7 @@ private:
/** Writes a single aligned leaf value */
void format_table_leaf_value(const dom::element& elem, const element_metrics& vm, size_t width,
table_column_type column_type, bool needs_comma,
- bool add_comma_space, size_t depth);
+ bool add_comma_space);
/** Whether, for a column of the given type, the comma goes right after the value */
bool comma_goes_before_padding(table_column_type column_type) const;
@@ -12963,7 +13695,7 @@ inline element_metrics structure_analyzer::analyze_array(const dom::array& arr,
for (dom::element child : arr) {
if (!first) {
- metrics.estimated_inline_len += 2; // ", "
+ metrics.estimated_inline_len += current_opts_->comma_padding ? 2 : 1; // ", " or ","
}
first = false;
@@ -13012,12 +13744,13 @@ inline element_metrics structure_analyzer::analyze_object(const dom::object& obj
for (dom::key_value_pair field : obj) {
if (!first) {
- metrics.estimated_inline_len += 2; // ", "
+ metrics.estimated_inline_len += current_opts_->comma_padding ? 2 : 1; // ", " or ","
}
first = false;
- // Key length: quotes + key + colon + space
- metrics.estimated_inline_len += estimate_string_length(field.key) + 2;
+ // Key length: quotes + key + colon (+ space)
+ metrics.estimated_inline_len += estimate_string_length(field.key) +
+ (current_opts_->colon_padding ? 2 : 1);
element_metrics child_metrics = analyze_element(field.value, depth + 1);
metrics.estimated_inline_len += child_metrics.estimated_inline_len;
@@ -13411,6 +14144,10 @@ simdjson_inline void fractured_formatter::print_newline() {
if (current_layout_ == layout_mode::single_line) {
return; // No newlines in inline mode
}
+ // Like the reference FracturedJson implementation, do not end a line with
+ // white space: the padding that aligns a value in a column is only needed
+ // when something follows it on the same line.
+ buffer.trim_trailing_spaces();
one_char('\n');
current_line_length_ = 0;
}
@@ -13688,7 +14425,7 @@ inline void fractured_string_builder::format_array_compact_multiline_aligned(
const element_metrics& row_metrics = child_metrics_at(metrics.children, child_idx);
if (columns.empty()) {
- format_table_scalar_row(elem, row_metrics, metrics.table_row_width_full, depth + 1,
+ format_table_scalar_row(elem, row_metrics, metrics.table_row_width_full,
metrics.scalar_column_type);
} else {
format_table_row(elem, row_metrics, columns, depth + 1);
@@ -13761,7 +14498,7 @@ inline void fractured_string_builder::format_table_row_columns(
} else {
const element_metrics& vm = child_metrics_at(value_metrics[col_idx]);
format_table_leaf_value(values[col_idx], vm, column.width, column.type, needs_comma,
- /*add_comma_space=*/true, depth);
+ /*add_comma_space=*/true);
}
if (!is_last_col && !needs_comma) {
@@ -13879,7 +14616,7 @@ inline bool fractured_string_builder::comma_goes_before_padding(table_column_typ
inline void fractured_string_builder::format_table_leaf_value(
const dom::element& elem, const element_metrics& vm, size_t width,
- table_column_type column_type, bool needs_comma, bool add_comma_space, size_t depth) {
+ table_column_type column_type, bool needs_comma, bool add_comma_space) {
bool comma_before_pad = needs_comma && comma_goes_before_padding(column_type);
bool comma_after_pad = needs_comma && !comma_before_pad;
@@ -13901,10 +14638,8 @@ inline void fractured_string_builder::format_table_leaf_value(
format_.one_char(' ');
}
- {
- scoped_single_line_mode single_line(format_);
- format_element(elem, vm, depth);
- }
+ // The cell shares its line with the rest of the row.
+ format_element_inline(elem, vm);
if (comma_before_pad) {
format_.comma();
@@ -13921,10 +14656,10 @@ inline void fractured_string_builder::format_table_leaf_value(
}
inline void fractured_string_builder::format_table_scalar_row(
- const dom::element& elem, const element_metrics& row_metrics, size_t width, size_t depth,
+ const dom::element& elem, const element_metrics& row_metrics, size_t width,
table_column_type column_type) {
format_table_leaf_value(elem, row_metrics, width, column_type,
- /*needs_comma=*/false, /*add_comma_space=*/false, depth);
+ /*needs_comma=*/false, /*add_comma_space=*/false);
}
inline void fractured_string_builder::format_array_as_table(const dom::array& arr,
@@ -13952,7 +14687,7 @@ inline void fractured_string_builder::format_array_as_table(const dom::array& ar
const element_metrics& row_metrics = child_metrics_at(metrics.children, child_idx);
if (columns.empty()) {
- format_table_scalar_row(elem, row_metrics, metrics.table_row_width, depth + 1,
+ format_table_scalar_row(elem, row_metrics, metrics.table_row_width,
metrics.scalar_column_type);
} else {
format_table_row(elem, row_metrics, columns, depth + 1);
@@ -44090,6 +44825,7 @@ consteval const char* get_json_key_name() {
#endif // SIMDJSON_ANNOTATIONS_H
/* end file simdjson/annotations.h */
/* skipped duplicate #include "simdjson/internal/jeaiii.h" */
+/* skipped duplicate #include "simdjson/internal/json_escape.h" */
#endif // SIMDJSON_GENERIC_BUILDER_DEPENDENCIES_H
/* end file simdjson/generic/builder/dependencies.h */
@@ -44964,6 +45700,7 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin
/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */
/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_H */
/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */
+/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */
/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */
namespace simdjson {
@@ -45000,6 +45737,9 @@ struct has_custom_serialization : std::false_type {};
namespace arm64 {
namespace builder {
+
+using simdjson::internal::escape_best::write_string_escaped;
+
/**
* A builder for JSON strings representing documents. This is a low-level
* builder that is not meant to be used directly by end-users. Though it
@@ -45346,6 +46086,7 @@ simdjson_warn_unused error_code to_json(const Z &z, std::string &s, size_t initi
/* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */
/* amalgamation skipped (editor-only): #include "simdjson/concepts.h" */
/* amalgamation skipped (editor-only): #include "simdjson/annotations.h" */
+/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */
/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */
#if SIMDJSON_STATIC_REFLECTION
@@ -45371,7 +46112,6 @@ namespace internal {
using simdjson::internal::write_uint_jeaiii;
simdjson_inline char *write_double(char *p, double v) noexcept;
} // namespace internal
-simdjson_really_inline size_t write_string_escaped(const std::string_view input, char *out);
SIMDJSON_PUSH_DISABLE_WARNINGS
SIMDJSON_DISABLE_GCC_WARNING(-Warray-bounds)
@@ -46392,619 +47132,13 @@ simdjson_warn_unused simdjson_result<std::string> extract_fractured_json(
/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_INL_H */
/* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */
/* amalgamation skipped (editor-only): #include "simdjson/internal/jeaiii.h" */
+/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */
/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */
-/*
- * Empirically, we have found that an inlined optimization is important for
- * performance. The following macros are not ideal. We should find a better
- * way to inline the code.
- */
-
-#if defined(__SSE2__) || defined(__x86_64__) || defined(__x86_64) || \
- (defined(_M_AMD64) || defined(_M_X64) || \
- (defined(_M_IX86_FP) && _M_IX86_FP == 2))
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_SSE2
-#define SIMDJSON_EXPERIMENTAL_HAS_SSE2 1
-#endif
-#endif
-
-#if defined(__aarch64__) || defined(_M_ARM64)
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_NEON
-#define SIMDJSON_EXPERIMENTAL_HAS_NEON 1
-#endif
-#endif
-#if defined(__loongarch_sx)
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_LSX
-#define SIMDJSON_EXPERIMENTAL_HAS_LSX 1
-#endif
-#endif
-#if defined(__loongarch_asx)
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_LASX
-#define SIMDJSON_EXPERIMENTAL_HAS_LASX 1
-#endif
-#endif
-#if defined(__riscv_v_intrinsic) && __riscv_v_intrinsic >= 11000 && \
- defined(__riscv_vector)
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_RVV
-#define SIMDJSON_EXPERIMENTAL_HAS_RVV 1
-#endif
-#endif
-#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) && defined(__POWER8_VECTOR__)
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64
-#define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1
-#endif
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_NEON
-#include <arm_neon.h>
-#ifdef _MSC_VER
-#include <intrin.h>
-#endif
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_SSE2
-#include <emmintrin.h>
-#if defined(__AVX2__)
-#include <immintrin.h>
-#endif
-#ifdef _MSC_VER
-#include <intrin.h>
-#endif
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_LSX
-#include <lsxintrin.h>
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_LASX
-#include <lasxintrin.h>
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_RVV
-#include <riscv_vector.h>
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_PPC64
-#include <altivec.h>
-#ifdef bool
-#undef bool
-#endif
-#ifdef vector
-#undef vector
-#endif
-#endif
-
-
namespace simdjson {
namespace arm64 {
namespace builder {
-static SIMDJSON_CONSTEXPR_LAMBDA std::array<uint8_t, 256>
- json_quotable_character = {
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
-
-/**
-
-A possible SWAR implementation of has_json_escapable_byte. It is not used
-because it is slower than the current implementation. It is kept here for
-reference (to show that we tried it).
-
-inline bool has_json_escapable_byte(uint64_t x) {
- uint64_t is_ascii = 0x8080808080808080ULL & ~x;
- uint64_t xor2 = x ^ 0x0202020202020202ULL;
- uint64_t lt32_or_eq34 = xor2 - 0x2121212121212121ULL;
- uint64_t sub92 = x ^ 0x5C5C5C5C5C5C5C5CULL;
- uint64_t eq92 = (sub92 - 0x0101010101010101ULL);
- return ((lt32_or_eq34 | eq92) & is_ascii) != 0;
-}
-
-**/
-
-#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 || SIMDJSON_EXPERIMENTAL_HAS_NEON
-#define SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE 1
-#endif
-
-// With block escaping (SSE2/NEON), write_string_escaped scans the input itself.
-#if !SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-// Scalar fallback for finding next quotable character
-SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline size_t
-find_next_json_quotable_character_scalar(const std::string_view view,
- size_t location) noexcept {
- for (auto pos = view.begin() + location; pos != view.end(); ++pos) {
- if (json_quotable_character[static_cast<uint8_t>(*pos)]) {
- return pos - view.begin();
- }
- }
- return size_t(view.size());
-}
-
-// SIMD-accelerated position finding that directly locates the first quotable
-// character, combining detection and position extraction in a single pass to
-// minimize redundant work.
-#if SIMDJSON_EXPERIMENTAL_HAS_LASX
-simdjson_inline size_t
-find_next_json_quotable_character(const std::string_view view,
- size_t location) noexcept {
- const size_t len = view.size();
- const uint8_t *ptr =
- reinterpret_cast<const uint8_t *>(view.data()) + location;
- size_t remaining = len - location;
-
- // SIMD constants for characters requiring escape
- __m256i v34 = __lasx_xvreplgr2vr_b(34); // '"'
- __m256i v92 = __lasx_xvreplgr2vr_b(92); // '\\'
- __m256i v32 = __lasx_xvreplgr2vr_b(32); // control char threshold
-
- while (remaining >= 32) {
- __m256i word = __lasx_xvld(ptr, 0);
-
- // Check for quotable characters: '"', '\\', or control chars (< 32)
- __m256i needs_escape = __lasx_xvseq_b(word, v34);
- needs_escape = __lasx_xvor_v(needs_escape, __lasx_xvseq_b(word, v92));
- needs_escape = __lasx_xvor_v(needs_escape, __lasx_xvslt_bu(word, v32));
-
- if (!__lasx_xbz_v(needs_escape)) {
- // Found a quotable character - locate it via the four 64-bit lanes
- uint64_t lane0 = __lasx_xvpickve2gr_du(needs_escape, 0);
- uint64_t lane1 = __lasx_xvpickve2gr_du(needs_escape, 1);
- uint64_t lane2 = __lasx_xvpickve2gr_du(needs_escape, 2);
- uint64_t lane3 = __lasx_xvpickve2gr_du(needs_escape, 3);
- size_t offset = ptr - reinterpret_cast<const uint8_t *>(view.data());
- if (lane0 != 0) {
- return offset + trailing_zeroes(lane0) / 8;
- } else if (lane1 != 0) {
- return offset + 8 + trailing_zeroes(lane1) / 8;
- } else if (lane2 != 0) {
- return offset + 16 + trailing_zeroes(lane2) / 8;
- } else {
- return offset + 24 + trailing_zeroes(lane3) / 8;
- }
- }
- ptr += 32;
- remaining -= 32;
- }
- size_t current = len - remaining;
- return find_next_json_quotable_character_scalar(view, current);
-}
-#elif SIMDJSON_EXPERIMENTAL_HAS_LSX
-simdjson_inline size_t
-find_next_json_quotable_character(const std::string_view view,
- size_t location) noexcept {
- const size_t len = view.size();
- const uint8_t *ptr =
- reinterpret_cast<const uint8_t *>(view.data()) + location;
- size_t remaining = len - location;
-
- //SIMD constants for characters requiring escape
- __m128i v34 = __lsx_vreplgr2vr_b(34); // '"'
- __m128i v92 = __lsx_vreplgr2vr_b(92); // '\\'
- __m128i v32 = __lsx_vreplgr2vr_b(32); // control char threshold
-
- while (remaining >= 16){
- __m128i word = __lsx_vld(ptr, 0);
-
- //Check for the quotable characters: '"', '\\', or control char (<32)
- __m128i needs_escape = __lsx_vseq_b(word, v34);
- needs_escape = __lsx_vor_v(needs_escape, __lsx_vseq_b(word, v92));
- needs_escape = __lsx_vor_v(needs_escape, __lsx_vslt_bu(word, v32));
-
- if (!__lsx_bz_v(needs_escape)){
-
- //Found quotable character - extract exact byte position
- uint64_t lo = __lsx_vpickve2gr_du(needs_escape,0);
- uint64_t hi = __lsx_vpickve2gr_du(needs_escape,1);
- size_t offset = ptr - reinterpret_cast<const uint8_t *>(view.data());
- if ( lo != 0) {
- return offset + trailing_zeroes(lo) / 8;
- } else {
- return offset + 8 + trailing_zeroes(hi) / 8;
- }
- }
- ptr += 16;
- remaining -= 16;
- }
- size_t current = len - remaining;
- return find_next_json_quotable_character_scalar(view, current);
-}
-#elif SIMDJSON_EXPERIMENTAL_HAS_RVV
-simdjson_inline size_t
-find_next_json_quotable_character(const std::string_view view,
- size_t location) noexcept {
- const size_t len = view.size();
- const uint8_t *ptr =
- reinterpret_cast<const uint8_t *>(view.data()) + location;
- size_t remaining = len - location;
-
- while (remaining > 0) {
- size_t vl = __riscv_vsetvl_e8m1(remaining);
- vuint8m1_t word = __riscv_vle8_v_u8m1(ptr, vl);
-
- // Check for quotable characters: '"', '\\', or control chars (< 32)
- vbool8_t needs_escape = __riscv_vmseq(word, (uint8_t)34, vl);
- needs_escape = __riscv_vmor(needs_escape,
- __riscv_vmseq(word, (uint8_t)92, vl), vl);
- needs_escape = __riscv_vmor(needs_escape,
- __riscv_vmsltu(word, (uint8_t)32, vl), vl);
-
- long first = __riscv_vfirst(needs_escape, vl);
- if (first >= 0) {
- size_t offset = ptr - reinterpret_cast<const uint8_t *>(view.data());
- return offset + first;
- }
- ptr += vl;
- remaining -= vl;
- }
-
- return len;
-}
-#elif SIMDJSON_EXPERIMENTAL_HAS_PPC64
-simdjson_inline size_t
-find_next_json_quotable_character(const std::string_view view,
- size_t location) noexcept {
- const size_t len = view.size();
- const uint8_t *ptr =
- reinterpret_cast<const uint8_t *>(view.data()) + location;
- size_t remaining = len - location;
-
- // SIMD constants for characters requiring escape
- __vector unsigned char v34 = vec_splats((unsigned char)34); // '"'
- __vector unsigned char v92 = vec_splats((unsigned char)92); // '\\'
- __vector unsigned char v32 = vec_splats((unsigned char)32); // control char threshold
-
- // Bitmask for vec_vbpermq to extract one bit per byte
- const __vector unsigned char perm_mask = {0x78, 0x70, 0x68, 0x60, 0x58, 0x50,
- 0x48, 0x40, 0x38, 0x30, 0x28, 0x20,
- 0x18, 0x10, 0x08, 0x00};
-
- while (remaining >= 16) {
- __vector unsigned char word =
- vec_vsx_ld(0, reinterpret_cast<const unsigned char *>(ptr));
-
- // Check for quotable characters: '"', '\\', or control chars (< 32)
- __vector unsigned char needs_escape =
- (__vector unsigned char)vec_cmpeq(word, v34);
- needs_escape = vec_or(needs_escape,
- (__vector unsigned char)vec_cmpeq(word, v92));
- needs_escape = vec_or(needs_escape,
- (__vector unsigned char)vec_cmplt(word, v32));
-
- __vector unsigned long long result =
- (__vector unsigned long long)vec_vbpermq(needs_escape, perm_mask);
-#ifdef __LITTLE_ENDIAN__
- unsigned int mask = static_cast<unsigned int>(result[1]);
-#else
- unsigned int mask = static_cast<unsigned int>(result[0]);
-#endif
- if (mask != 0) {
- size_t offset = ptr - reinterpret_cast<const uint8_t *>(view.data());
- return offset + __builtin_ctz(mask);
- }
- ptr += 16;
- remaining -= 16;
- }
-
- // Scalar fallback for remaining bytes
- size_t current = len - remaining;
- return find_next_json_quotable_character_scalar(view, current);
-}
-#else
-SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline size_t
-find_next_json_quotable_character(const std::string_view view,
- size_t location) noexcept {
- return find_next_json_quotable_character_scalar(view, location);
-}
-#endif
-#endif // !SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-
-SIMDJSON_CONSTEXPR_LAMBDA static std::string_view control_chars[] = {
- "\\u0000", "\\u0001", "\\u0002", "\\u0003", "\\u0004", "\\u0005", "\\u0006",
- "\\u0007", "\\b", "\\t", "\\n", "\\u000b", "\\f", "\\r",
- "\\u000e", "\\u000f", "\\u0010", "\\u0011", "\\u0012", "\\u0013", "\\u0014",
- "\\u0015", "\\u0016", "\\u0017", "\\u0018", "\\u0019", "\\u001a", "\\u001b",
- "\\u001c", "\\u001d", "\\u001e", "\\u001f"};
-
-// All Unicode characters may be placed within the quotation marks, except for
-// the characters that MUST be escaped: quotation mark, reverse solidus, and the
-// control characters (U+0000 through U+001F). There are two-character sequence
-// escape representations of some popular characters:
-// \", \\, \b, \f, \n, \r, \t.
-SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline void escape_json_char(char c, char *&out) {
- if (c == '"') {
- memcpy(out, "\\\"", 2);
- out += 2;
- } else if (c == '\\') {
- memcpy(out, "\\\\", 2);
- out += 2;
- } else {
- std::string_view v = control_chars[uint8_t(c)];
- memcpy(out, v.data(), v.size());
- out += v.size();
- }
-}
-
-#if SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-
-#if SIMDJSON_EXPERIMENTAL_HAS_SSE2
-
-using escape_vector = __m128i;
-// Mask bits that each input byte contributes to escape_bitmask().
-static constexpr unsigned escape_mask_bits = 1;
-
-simdjson_inline escape_vector escape_load16(const uint8_t *p) noexcept {
- return _mm_loadu_si128(reinterpret_cast<const __m128i *>(p));
-}
-
-simdjson_inline void escape_store16(char *out, escape_vector v) noexcept {
- _mm_storeu_si128(reinterpret_cast<__m128i *>(out), v);
-}
-
-// Builds a vector whose bytes 0..7 come from a and bytes 8..15 from b.
-simdjson_inline escape_vector escape_load8x2(const uint8_t *a,
- const uint8_t *b) noexcept {
- return _mm_unpacklo_epi64(
- _mm_loadl_epi64(reinterpret_cast<const __m128i *>(a)),
- _mm_loadl_epi64(reinterpret_cast<const __m128i *>(b)));
-}
-
-// Builds a vector whose bytes 0..3 come from a and bytes 4..7 from b. The
-// upper half is unspecified; callers only look at the low eight bits.
-simdjson_inline escape_vector escape_load4x2(const uint8_t *a,
- const uint8_t *b) noexcept {
- int32_t a32, b32;
- memcpy(&a32, a, 4);
- memcpy(&b32, b, 4);
- return _mm_unpacklo_epi32(_mm_cvtsi32_si128(a32), _mm_cvtsi32_si128(b32));
-}
-
-// Sets every bit of byte k when byte k of v is a quotable character ('"',
-// '\\' or a control character).
-simdjson_inline escape_vector escape_flags(escape_vector v) noexcept {
- const __m128i v34 = _mm_set1_epi8(34); // '"'
- const __m128i v92 = _mm_set1_epi8(92); // '\\'
- const __m128i v31 = _mm_set1_epi8(31); // for control char detection
- __m128i needs_escape = _mm_cmpeq_epi8(v, v34);
- needs_escape = _mm_or_si128(needs_escape, _mm_cmpeq_epi8(v, v92));
- return _mm_or_si128(
- needs_escape, _mm_cmpeq_epi8(_mm_subs_epu8(v, v31), _mm_setzero_si128()));
-}
-
-// True when any byte needs escaping. Kept separate from escape_bitmask
-// because some instruction sets can answer it without leaving the vector
-// register file; on SSE2 the compiler folds the two together.
-simdjson_inline bool escape_any(escape_vector flags) noexcept {
- return _mm_movemask_epi8(flags) != 0;
-}
-
-// A 16-bit mask with bit k set when byte k needed escaping.
-simdjson_inline uint64_t escape_bitmask(escape_vector flags) noexcept {
- return uint64_t(uint32_t(_mm_movemask_epi8(flags)));
-}
-
-#else // SIMDJSON_EXPERIMENTAL_HAS_NEON
-
-using escape_vector = uint8x16_t;
-static constexpr unsigned escape_mask_bits = 4;
-
-simdjson_inline escape_vector escape_load16(const uint8_t *p) noexcept {
- return vld1q_u8(p);
-}
-
-simdjson_inline void escape_store16(char *out, escape_vector v) noexcept {
- vst1q_u8(reinterpret_cast<uint8_t *>(out), v);
-}
-
-simdjson_inline escape_vector escape_load8x2(const uint8_t *a,
- const uint8_t *b) noexcept {
- uint64_t a64, b64;
- memcpy(&a64, a, 8);
- memcpy(&b64, b, 8);
- return vreinterpretq_u8_u64(vsetq_lane_u64(b64, vdupq_n_u64(a64), 1));
-}
-
-simdjson_inline escape_vector escape_load4x2(const uint8_t *a,
- const uint8_t *b) noexcept {
- uint32_t a32, b32;
- memcpy(&a32, a, 4);
- memcpy(&b32, b, 4);
- return vreinterpretq_u8_u32(vsetq_lane_u32(b32, vdupq_n_u32(a32), 1));
-}
-
-simdjson_inline escape_vector escape_flags(escape_vector v) noexcept {
- uint8x16_t needs_escape = vceqq_u8(v, vdupq_n_u8(34)); // '"'
- needs_escape = vorrq_u8(needs_escape, vceqq_u8(v, vdupq_n_u8(92))); // '\\'
- return vorrq_u8(needs_escape, vcltq_u8(v, vdupq_n_u8(32)));
-}
-
-simdjson_inline bool escape_any(escape_vector flags) noexcept {
- return vmaxvq_u32(vreinterpretq_u32_u8(flags)) != 0;
-}
-
-// Four bits per byte rather than one: escape_block and the tail paths scale
-// their shifts by escape_mask_bits to match.
-simdjson_inline uint64_t escape_bitmask(escape_vector flags) noexcept {
- uint8x8_t narrowed = vshrn_n_u16(vreinterpretq_u16_u8(flags), 4);
- return vget_lane_u64(vreinterpret_u64_u8(narrowed), 0);
-}
-
-#endif // instruction set selection
-
-// The escape bitmask of a 16-byte block.
-simdjson_inline uint64_t escape_mask(escape_vector v) noexcept {
- return escape_bitmask(escape_flags(v));
-}
-
-// Copies n bytes with n < 16, using overlapping loads and stores. It never
-// reads more than n bytes from src, nor writes more than n bytes to dst.
-simdjson_inline void copy_lt16(char *dst, const uint8_t *src,
- size_t n) noexcept {
- if (n >= 8) {
- memcpy(dst, src, 8);
- memcpy(dst + n - 8, src + n - 8, 8);
- } else if (n >= 4) {
- memcpy(dst, src, 4);
- memcpy(dst + n - 4, src + n - 4, 4);
- } else if (n > 0) {
- dst[0] = char(src[0]);
- dst[n >> 1] = char(src[n >> 1]);
- dst[n - 1] = char(src[n - 1]);
- }
-}
-
-// Escapes the bytes of src in the range [i, blockend), given that m is the
-// (non-zero) escape mask for that range: the escape_mask_bits-wide lane of m
-// at byte k is non-zero when src[i + k] requires escaping. Returns the updated
-// output pointer.
-simdjson_never_inline char *escape_block(const uint8_t *src, char *out,
- size_t i, size_t blockend,
- uint64_t m) noexcept {
- constexpr uint64_t lane = (uint64_t(1) << escape_mask_bits) - 1;
-
- size_t pos = i; // first byte not yet copied
- while (m) {
- const size_t tz = trailing_zeroes(m);
- const size_t next = i + tz / escape_mask_bits;
- // Copy the run of safe bytes that precedes this escape.
- copy_lt16(out, src + pos, next - pos);
- out += next - pos;
- escape_json_char(char(src[next]), out);
- pos = next + 1;
- m &= ~(lane << tz);
- }
- // Copy whatever follows the last escape.
- copy_lt16(out, src + pos, blockend - pos);
- return out + (blockend - pos);
-}
-
-// Writes the escaped version of input to out, returning the number of bytes
-// written.
-simdjson_really_inline size_t write_string_escaped(const std::string_view input, char *out) {
- const size_t len = input.size();
- const uint8_t *src = reinterpret_cast<const uint8_t *>(input.data());
- const char *const initout = out;
-
- size_t i = 0;
-#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 && defined(__AVX2__)
- while (i + 32 <= len) {
- const __m256i word = _mm256_loadu_si256(reinterpret_cast<const __m256i *>(src + i));
- const __m256i flags = _mm256_or_si256(
- _mm256_or_si256(_mm256_cmpeq_epi8(word, _mm256_set1_epi8(34)), // '"'
- _mm256_cmpeq_epi8(word, _mm256_set1_epi8(92))), // '\\'
- _mm256_cmpeq_epi8(_mm256_subs_epu8(word, _mm256_set1_epi8(31)),
- _mm256_setzero_si256())); // control
- const uint32_t mask = uint32_t(_mm256_movemask_epi8(flags));
- if (simdjson_likely(mask == 0)) {
- _mm256_storeu_si256(reinterpret_cast<__m256i *>(out), word);
- out += 32;
- } else {
- for (size_t half = 0; half < 32; half += 16) {
- const uint64_t m = (mask >> half) & 0xFFFF;
- if (m == 0) {
- escape_store16(out, escape_load16(src + i + half));
- out += 16;
- } else {
- out = escape_block(src, out, i + half, i + half + 16, m);
- }
- }
- }
- i += 32;
- }
-#endif
- while (i + 16 <= len) {
- escape_vector word = escape_load16(src + i);
- escape_vector flags = escape_flags(word);
- if (simdjson_likely(!escape_any(flags))) {
- escape_store16(out, word);
- out += 16;
- } else {
- out = escape_block(src, out, i, i + 16, escape_bitmask(flags));
- }
- i += 16;
- }
- if (i < len) {
- const size_t rem = len - i;
- uint64_t m;
- if (len >= 16) {
- // The last 16 bytes of the input are in bounds. Bit k of that block's
- // mask belongs to input position len - 16 + k, so shift it down to align
- // bit 0 with position i.
- m = escape_mask(escape_load16(src + len - 16)) >>
- (escape_mask_bits * (16 - rem));
- } else if (len >= 8) {
- // Here i == 0 and rem == len. Two overlapping 8-byte loads cover
- // [0, 8) and [len - 8, len), which is the whole input since len < 16.
- uint64_t mm = escape_mask(escape_load8x2(src, src + len - 8));
- constexpr uint64_t low8 = (uint64_t(1) << (escape_mask_bits * 8)) - 1;
- m = (mm & low8) |
- ((mm >> (escape_mask_bits * 8)) << (escape_mask_bits * (len - 8)));
- } else if (len >= 4) {
- // Same idea with two overlapping 4-byte loads.
- uint64_t mm = escape_mask(escape_load4x2(src, src + len - 4));
- constexpr uint64_t low4 = (uint64_t(1) << (escape_mask_bits * 4)) - 1;
- m = (mm & low4) | (((mm >> (escape_mask_bits * 4)) & low4)
- << (escape_mask_bits * (len - 4)));
- } else {
- // Fewer than 4 bytes: at most three table lookups, no need for SIMD.
- for (size_t k = 0; k < len; k++) {
- uint8_t c = src[k];
- if (json_quotable_character[c]) {
- escape_json_char(char(c), out);
- } else {
- *out++ = char(c);
- }
- }
- return size_t(out - initout);
- }
- if (m == 0) {
- copy_lt16(out, src + i, rem);
- out += rem;
- } else {
- out = escape_block(src, out, i, len, m);
- }
- }
- return size_t(out - initout);
-}
-
-#else // SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-
-// Writes the escaped version of input to out, returning the number of bytes
-// written. Uses SIMD position finding to locate quotable characters efficiently.
-inline size_t write_string_escaped(const std::string_view input, char *out) {
- size_t mysize = input.size();
-
- // Use SIMD position finder directly - it returns mysize if no escape needed
- size_t location = find_next_json_quotable_character(input, 0);
- if (location == mysize) {
- // Fast path: no escaping needed
- memcpy(out, input.data(), input.size());
- return input.size();
- }
-
- const char *const initout = out;
- memcpy(out, input.data(), location);
- out += location;
- escape_json_char(input[location], out);
- location += 1;
- while (location < mysize) {
- size_t newlocation = find_next_json_quotable_character(input, location);
- memcpy(out, input.data() + location, newlocation - location);
- out += newlocation - location;
- location = newlocation;
- if (location == mysize) {
- break;
- }
- escape_json_char(input[location], out);
- location += 1;
- }
- return size_t(out - initout);
-}
-
-#endif // SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-#undef SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-
namespace internal {
// Sets the size of s to n without initializing the new bytes when the
@@ -47859,6 +47993,7 @@ simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t
/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */
/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_H */
/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */
+/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */
/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */
namespace simdjson {
@@ -47895,6 +48030,9 @@ struct has_custom_serialization : std::false_type {};
namespace fallback {
namespace builder {
+
+using simdjson::internal::escape_best::write_string_escaped;
+
/**
* A builder for JSON strings representing documents. This is a low-level
* builder that is not meant to be used directly by end-users. Though it
@@ -48241,6 +48379,7 @@ simdjson_warn_unused error_code to_json(const Z &z, std::string &s, size_t initi
/* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */
/* amalgamation skipped (editor-only): #include "simdjson/concepts.h" */
/* amalgamation skipped (editor-only): #include "simdjson/annotations.h" */
+/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */
/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */
#if SIMDJSON_STATIC_REFLECTION
@@ -48266,7 +48405,6 @@ namespace internal {
using simdjson::internal::write_uint_jeaiii;
simdjson_inline char *write_double(char *p, double v) noexcept;
} // namespace internal
-simdjson_really_inline size_t write_string_escaped(const std::string_view input, char *out);
SIMDJSON_PUSH_DISABLE_WARNINGS
SIMDJSON_DISABLE_GCC_WARNING(-Warray-bounds)
@@ -49287,619 +49425,13 @@ simdjson_warn_unused simdjson_result<std::string> extract_fractured_json(
/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_INL_H */
/* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */
/* amalgamation skipped (editor-only): #include "simdjson/internal/jeaiii.h" */
+/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */
/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */
-/*
- * Empirically, we have found that an inlined optimization is important for
- * performance. The following macros are not ideal. We should find a better
- * way to inline the code.
- */
-
-#if defined(__SSE2__) || defined(__x86_64__) || defined(__x86_64) || \
- (defined(_M_AMD64) || defined(_M_X64) || \
- (defined(_M_IX86_FP) && _M_IX86_FP == 2))
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_SSE2
-#define SIMDJSON_EXPERIMENTAL_HAS_SSE2 1
-#endif
-#endif
-
-#if defined(__aarch64__) || defined(_M_ARM64)
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_NEON
-#define SIMDJSON_EXPERIMENTAL_HAS_NEON 1
-#endif
-#endif
-#if defined(__loongarch_sx)
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_LSX
-#define SIMDJSON_EXPERIMENTAL_HAS_LSX 1
-#endif
-#endif
-#if defined(__loongarch_asx)
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_LASX
-#define SIMDJSON_EXPERIMENTAL_HAS_LASX 1
-#endif
-#endif
-#if defined(__riscv_v_intrinsic) && __riscv_v_intrinsic >= 11000 && \
- defined(__riscv_vector)
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_RVV
-#define SIMDJSON_EXPERIMENTAL_HAS_RVV 1
-#endif
-#endif
-#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) && defined(__POWER8_VECTOR__)
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64
-#define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1
-#endif
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_NEON
-#include <arm_neon.h>
-#ifdef _MSC_VER
-#include <intrin.h>
-#endif
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_SSE2
-#include <emmintrin.h>
-#if defined(__AVX2__)
-#include <immintrin.h>
-#endif
-#ifdef _MSC_VER
-#include <intrin.h>
-#endif
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_LSX
-#include <lsxintrin.h>
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_LASX
-#include <lasxintrin.h>
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_RVV
-#include <riscv_vector.h>
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_PPC64
-#include <altivec.h>
-#ifdef bool
-#undef bool
-#endif
-#ifdef vector
-#undef vector
-#endif
-#endif
-
-
namespace simdjson {
namespace fallback {
namespace builder {
-static SIMDJSON_CONSTEXPR_LAMBDA std::array<uint8_t, 256>
- json_quotable_character = {
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
-
-/**
-
-A possible SWAR implementation of has_json_escapable_byte. It is not used
-because it is slower than the current implementation. It is kept here for
-reference (to show that we tried it).
-
-inline bool has_json_escapable_byte(uint64_t x) {
- uint64_t is_ascii = 0x8080808080808080ULL & ~x;
- uint64_t xor2 = x ^ 0x0202020202020202ULL;
- uint64_t lt32_or_eq34 = xor2 - 0x2121212121212121ULL;
- uint64_t sub92 = x ^ 0x5C5C5C5C5C5C5C5CULL;
- uint64_t eq92 = (sub92 - 0x0101010101010101ULL);
- return ((lt32_or_eq34 | eq92) & is_ascii) != 0;
-}
-
-**/
-
-#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 || SIMDJSON_EXPERIMENTAL_HAS_NEON
-#define SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE 1
-#endif
-
-// With block escaping (SSE2/NEON), write_string_escaped scans the input itself.
-#if !SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-// Scalar fallback for finding next quotable character
-SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline size_t
-find_next_json_quotable_character_scalar(const std::string_view view,
- size_t location) noexcept {
- for (auto pos = view.begin() + location; pos != view.end(); ++pos) {
- if (json_quotable_character[static_cast<uint8_t>(*pos)]) {
- return pos - view.begin();
- }
- }
- return size_t(view.size());
-}
-
-// SIMD-accelerated position finding that directly locates the first quotable
-// character, combining detection and position extraction in a single pass to
-// minimize redundant work.
-#if SIMDJSON_EXPERIMENTAL_HAS_LASX
-simdjson_inline size_t
-find_next_json_quotable_character(const std::string_view view,
- size_t location) noexcept {
- const size_t len = view.size();
- const uint8_t *ptr =
- reinterpret_cast<const uint8_t *>(view.data()) + location;
- size_t remaining = len - location;
-
- // SIMD constants for characters requiring escape
- __m256i v34 = __lasx_xvreplgr2vr_b(34); // '"'
- __m256i v92 = __lasx_xvreplgr2vr_b(92); // '\\'
- __m256i v32 = __lasx_xvreplgr2vr_b(32); // control char threshold
-
- while (remaining >= 32) {
- __m256i word = __lasx_xvld(ptr, 0);
-
- // Check for quotable characters: '"', '\\', or control chars (< 32)
- __m256i needs_escape = __lasx_xvseq_b(word, v34);
- needs_escape = __lasx_xvor_v(needs_escape, __lasx_xvseq_b(word, v92));
- needs_escape = __lasx_xvor_v(needs_escape, __lasx_xvslt_bu(word, v32));
-
- if (!__lasx_xbz_v(needs_escape)) {
- // Found a quotable character - locate it via the four 64-bit lanes
- uint64_t lane0 = __lasx_xvpickve2gr_du(needs_escape, 0);
- uint64_t lane1 = __lasx_xvpickve2gr_du(needs_escape, 1);
- uint64_t lane2 = __lasx_xvpickve2gr_du(needs_escape, 2);
- uint64_t lane3 = __lasx_xvpickve2gr_du(needs_escape, 3);
- size_t offset = ptr - reinterpret_cast<const uint8_t *>(view.data());
- if (lane0 != 0) {
- return offset + trailing_zeroes(lane0) / 8;
- } else if (lane1 != 0) {
- return offset + 8 + trailing_zeroes(lane1) / 8;
- } else if (lane2 != 0) {
- return offset + 16 + trailing_zeroes(lane2) / 8;
- } else {
- return offset + 24 + trailing_zeroes(lane3) / 8;
- }
- }
- ptr += 32;
- remaining -= 32;
- }
- size_t current = len - remaining;
- return find_next_json_quotable_character_scalar(view, current);
-}
-#elif SIMDJSON_EXPERIMENTAL_HAS_LSX
-simdjson_inline size_t
-find_next_json_quotable_character(const std::string_view view,
- size_t location) noexcept {
- const size_t len = view.size();
- const uint8_t *ptr =
- reinterpret_cast<const uint8_t *>(view.data()) + location;
- size_t remaining = len - location;
-
- //SIMD constants for characters requiring escape
- __m128i v34 = __lsx_vreplgr2vr_b(34); // '"'
- __m128i v92 = __lsx_vreplgr2vr_b(92); // '\\'
- __m128i v32 = __lsx_vreplgr2vr_b(32); // control char threshold
-
- while (remaining >= 16){
- __m128i word = __lsx_vld(ptr, 0);
-
- //Check for the quotable characters: '"', '\\', or control char (<32)
- __m128i needs_escape = __lsx_vseq_b(word, v34);
- needs_escape = __lsx_vor_v(needs_escape, __lsx_vseq_b(word, v92));
- needs_escape = __lsx_vor_v(needs_escape, __lsx_vslt_bu(word, v32));
-
- if (!__lsx_bz_v(needs_escape)){
-
- //Found quotable character - extract exact byte position
- uint64_t lo = __lsx_vpickve2gr_du(needs_escape,0);
- uint64_t hi = __lsx_vpickve2gr_du(needs_escape,1);
- size_t offset = ptr - reinterpret_cast<const uint8_t *>(view.data());
- if ( lo != 0) {
- return offset + trailing_zeroes(lo) / 8;
- } else {
- return offset + 8 + trailing_zeroes(hi) / 8;
- }
- }
- ptr += 16;
- remaining -= 16;
- }
- size_t current = len - remaining;
- return find_next_json_quotable_character_scalar(view, current);
-}
-#elif SIMDJSON_EXPERIMENTAL_HAS_RVV
-simdjson_inline size_t
-find_next_json_quotable_character(const std::string_view view,
- size_t location) noexcept {
- const size_t len = view.size();
- const uint8_t *ptr =
- reinterpret_cast<const uint8_t *>(view.data()) + location;
- size_t remaining = len - location;
-
- while (remaining > 0) {
- size_t vl = __riscv_vsetvl_e8m1(remaining);
- vuint8m1_t word = __riscv_vle8_v_u8m1(ptr, vl);
-
- // Check for quotable characters: '"', '\\', or control chars (< 32)
- vbool8_t needs_escape = __riscv_vmseq(word, (uint8_t)34, vl);
- needs_escape = __riscv_vmor(needs_escape,
- __riscv_vmseq(word, (uint8_t)92, vl), vl);
- needs_escape = __riscv_vmor(needs_escape,
- __riscv_vmsltu(word, (uint8_t)32, vl), vl);
-
- long first = __riscv_vfirst(needs_escape, vl);
- if (first >= 0) {
- size_t offset = ptr - reinterpret_cast<const uint8_t *>(view.data());
- return offset + first;
- }
- ptr += vl;
- remaining -= vl;
- }
-
- return len;
-}
-#elif SIMDJSON_EXPERIMENTAL_HAS_PPC64
-simdjson_inline size_t
-find_next_json_quotable_character(const std::string_view view,
- size_t location) noexcept {
- const size_t len = view.size();
- const uint8_t *ptr =
- reinterpret_cast<const uint8_t *>(view.data()) + location;
- size_t remaining = len - location;
-
- // SIMD constants for characters requiring escape
- __vector unsigned char v34 = vec_splats((unsigned char)34); // '"'
- __vector unsigned char v92 = vec_splats((unsigned char)92); // '\\'
- __vector unsigned char v32 = vec_splats((unsigned char)32); // control char threshold
-
- // Bitmask for vec_vbpermq to extract one bit per byte
- const __vector unsigned char perm_mask = {0x78, 0x70, 0x68, 0x60, 0x58, 0x50,
- 0x48, 0x40, 0x38, 0x30, 0x28, 0x20,
- 0x18, 0x10, 0x08, 0x00};
-
- while (remaining >= 16) {
- __vector unsigned char word =
- vec_vsx_ld(0, reinterpret_cast<const unsigned char *>(ptr));
-
- // Check for quotable characters: '"', '\\', or control chars (< 32)
- __vector unsigned char needs_escape =
- (__vector unsigned char)vec_cmpeq(word, v34);
- needs_escape = vec_or(needs_escape,
- (__vector unsigned char)vec_cmpeq(word, v92));
- needs_escape = vec_or(needs_escape,
- (__vector unsigned char)vec_cmplt(word, v32));
-
- __vector unsigned long long result =
- (__vector unsigned long long)vec_vbpermq(needs_escape, perm_mask);
-#ifdef __LITTLE_ENDIAN__
- unsigned int mask = static_cast<unsigned int>(result[1]);
-#else
- unsigned int mask = static_cast<unsigned int>(result[0]);
-#endif
- if (mask != 0) {
- size_t offset = ptr - reinterpret_cast<const uint8_t *>(view.data());
- return offset + __builtin_ctz(mask);
- }
- ptr += 16;
- remaining -= 16;
- }
-
- // Scalar fallback for remaining bytes
- size_t current = len - remaining;
- return find_next_json_quotable_character_scalar(view, current);
-}
-#else
-SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline size_t
-find_next_json_quotable_character(const std::string_view view,
- size_t location) noexcept {
- return find_next_json_quotable_character_scalar(view, location);
-}
-#endif
-#endif // !SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-
-SIMDJSON_CONSTEXPR_LAMBDA static std::string_view control_chars[] = {
- "\\u0000", "\\u0001", "\\u0002", "\\u0003", "\\u0004", "\\u0005", "\\u0006",
- "\\u0007", "\\b", "\\t", "\\n", "\\u000b", "\\f", "\\r",
- "\\u000e", "\\u000f", "\\u0010", "\\u0011", "\\u0012", "\\u0013", "\\u0014",
- "\\u0015", "\\u0016", "\\u0017", "\\u0018", "\\u0019", "\\u001a", "\\u001b",
- "\\u001c", "\\u001d", "\\u001e", "\\u001f"};
-
-// All Unicode characters may be placed within the quotation marks, except for
-// the characters that MUST be escaped: quotation mark, reverse solidus, and the
-// control characters (U+0000 through U+001F). There are two-character sequence
-// escape representations of some popular characters:
-// \", \\, \b, \f, \n, \r, \t.
-SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline void escape_json_char(char c, char *&out) {
- if (c == '"') {
- memcpy(out, "\\\"", 2);
- out += 2;
- } else if (c == '\\') {
- memcpy(out, "\\\\", 2);
- out += 2;
- } else {
- std::string_view v = control_chars[uint8_t(c)];
- memcpy(out, v.data(), v.size());
- out += v.size();
- }
-}
-
-#if SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-
-#if SIMDJSON_EXPERIMENTAL_HAS_SSE2
-
-using escape_vector = __m128i;
-// Mask bits that each input byte contributes to escape_bitmask().
-static constexpr unsigned escape_mask_bits = 1;
-
-simdjson_inline escape_vector escape_load16(const uint8_t *p) noexcept {
- return _mm_loadu_si128(reinterpret_cast<const __m128i *>(p));
-}
-
-simdjson_inline void escape_store16(char *out, escape_vector v) noexcept {
- _mm_storeu_si128(reinterpret_cast<__m128i *>(out), v);
-}
-
-// Builds a vector whose bytes 0..7 come from a and bytes 8..15 from b.
-simdjson_inline escape_vector escape_load8x2(const uint8_t *a,
- const uint8_t *b) noexcept {
- return _mm_unpacklo_epi64(
- _mm_loadl_epi64(reinterpret_cast<const __m128i *>(a)),
- _mm_loadl_epi64(reinterpret_cast<const __m128i *>(b)));
-}
-
-// Builds a vector whose bytes 0..3 come from a and bytes 4..7 from b. The
-// upper half is unspecified; callers only look at the low eight bits.
-simdjson_inline escape_vector escape_load4x2(const uint8_t *a,
- const uint8_t *b) noexcept {
- int32_t a32, b32;
- memcpy(&a32, a, 4);
- memcpy(&b32, b, 4);
- return _mm_unpacklo_epi32(_mm_cvtsi32_si128(a32), _mm_cvtsi32_si128(b32));
-}
-
-// Sets every bit of byte k when byte k of v is a quotable character ('"',
-// '\\' or a control character).
-simdjson_inline escape_vector escape_flags(escape_vector v) noexcept {
- const __m128i v34 = _mm_set1_epi8(34); // '"'
- const __m128i v92 = _mm_set1_epi8(92); // '\\'
- const __m128i v31 = _mm_set1_epi8(31); // for control char detection
- __m128i needs_escape = _mm_cmpeq_epi8(v, v34);
- needs_escape = _mm_or_si128(needs_escape, _mm_cmpeq_epi8(v, v92));
- return _mm_or_si128(
- needs_escape, _mm_cmpeq_epi8(_mm_subs_epu8(v, v31), _mm_setzero_si128()));
-}
-
-// True when any byte needs escaping. Kept separate from escape_bitmask
-// because some instruction sets can answer it without leaving the vector
-// register file; on SSE2 the compiler folds the two together.
-simdjson_inline bool escape_any(escape_vector flags) noexcept {
- return _mm_movemask_epi8(flags) != 0;
-}
-
-// A 16-bit mask with bit k set when byte k needed escaping.
-simdjson_inline uint64_t escape_bitmask(escape_vector flags) noexcept {
- return uint64_t(uint32_t(_mm_movemask_epi8(flags)));
-}
-
-#else // SIMDJSON_EXPERIMENTAL_HAS_NEON
-
-using escape_vector = uint8x16_t;
-static constexpr unsigned escape_mask_bits = 4;
-
-simdjson_inline escape_vector escape_load16(const uint8_t *p) noexcept {
- return vld1q_u8(p);
-}
-
-simdjson_inline void escape_store16(char *out, escape_vector v) noexcept {
- vst1q_u8(reinterpret_cast<uint8_t *>(out), v);
-}
-
-simdjson_inline escape_vector escape_load8x2(const uint8_t *a,
- const uint8_t *b) noexcept {
- uint64_t a64, b64;
- memcpy(&a64, a, 8);
- memcpy(&b64, b, 8);
- return vreinterpretq_u8_u64(vsetq_lane_u64(b64, vdupq_n_u64(a64), 1));
-}
-
-simdjson_inline escape_vector escape_load4x2(const uint8_t *a,
- const uint8_t *b) noexcept {
- uint32_t a32, b32;
- memcpy(&a32, a, 4);
- memcpy(&b32, b, 4);
- return vreinterpretq_u8_u32(vsetq_lane_u32(b32, vdupq_n_u32(a32), 1));
-}
-
-simdjson_inline escape_vector escape_flags(escape_vector v) noexcept {
- uint8x16_t needs_escape = vceqq_u8(v, vdupq_n_u8(34)); // '"'
- needs_escape = vorrq_u8(needs_escape, vceqq_u8(v, vdupq_n_u8(92))); // '\\'
- return vorrq_u8(needs_escape, vcltq_u8(v, vdupq_n_u8(32)));
-}
-
-simdjson_inline bool escape_any(escape_vector flags) noexcept {
- return vmaxvq_u32(vreinterpretq_u32_u8(flags)) != 0;
-}
-
-// Four bits per byte rather than one: escape_block and the tail paths scale
-// their shifts by escape_mask_bits to match.
-simdjson_inline uint64_t escape_bitmask(escape_vector flags) noexcept {
- uint8x8_t narrowed = vshrn_n_u16(vreinterpretq_u16_u8(flags), 4);
- return vget_lane_u64(vreinterpret_u64_u8(narrowed), 0);
-}
-
-#endif // instruction set selection
-
-// The escape bitmask of a 16-byte block.
-simdjson_inline uint64_t escape_mask(escape_vector v) noexcept {
- return escape_bitmask(escape_flags(v));
-}
-
-// Copies n bytes with n < 16, using overlapping loads and stores. It never
-// reads more than n bytes from src, nor writes more than n bytes to dst.
-simdjson_inline void copy_lt16(char *dst, const uint8_t *src,
- size_t n) noexcept {
- if (n >= 8) {
- memcpy(dst, src, 8);
- memcpy(dst + n - 8, src + n - 8, 8);
- } else if (n >= 4) {
- memcpy(dst, src, 4);
- memcpy(dst + n - 4, src + n - 4, 4);
- } else if (n > 0) {
- dst[0] = char(src[0]);
- dst[n >> 1] = char(src[n >> 1]);
- dst[n - 1] = char(src[n - 1]);
- }
-}
-
-// Escapes the bytes of src in the range [i, blockend), given that m is the
-// (non-zero) escape mask for that range: the escape_mask_bits-wide lane of m
-// at byte k is non-zero when src[i + k] requires escaping. Returns the updated
-// output pointer.
-simdjson_never_inline char *escape_block(const uint8_t *src, char *out,
- size_t i, size_t blockend,
- uint64_t m) noexcept {
- constexpr uint64_t lane = (uint64_t(1) << escape_mask_bits) - 1;
-
- size_t pos = i; // first byte not yet copied
- while (m) {
- const size_t tz = trailing_zeroes(m);
- const size_t next = i + tz / escape_mask_bits;
- // Copy the run of safe bytes that precedes this escape.
- copy_lt16(out, src + pos, next - pos);
- out += next - pos;
- escape_json_char(char(src[next]), out);
- pos = next + 1;
- m &= ~(lane << tz);
- }
- // Copy whatever follows the last escape.
- copy_lt16(out, src + pos, blockend - pos);
- return out + (blockend - pos);
-}
-
-// Writes the escaped version of input to out, returning the number of bytes
-// written.
-simdjson_really_inline size_t write_string_escaped(const std::string_view input, char *out) {
- const size_t len = input.size();
- const uint8_t *src = reinterpret_cast<const uint8_t *>(input.data());
- const char *const initout = out;
-
- size_t i = 0;
-#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 && defined(__AVX2__)
- while (i + 32 <= len) {
- const __m256i word = _mm256_loadu_si256(reinterpret_cast<const __m256i *>(src + i));
- const __m256i flags = _mm256_or_si256(
- _mm256_or_si256(_mm256_cmpeq_epi8(word, _mm256_set1_epi8(34)), // '"'
- _mm256_cmpeq_epi8(word, _mm256_set1_epi8(92))), // '\\'
- _mm256_cmpeq_epi8(_mm256_subs_epu8(word, _mm256_set1_epi8(31)),
- _mm256_setzero_si256())); // control
- const uint32_t mask = uint32_t(_mm256_movemask_epi8(flags));
- if (simdjson_likely(mask == 0)) {
- _mm256_storeu_si256(reinterpret_cast<__m256i *>(out), word);
- out += 32;
- } else {
- for (size_t half = 0; half < 32; half += 16) {
- const uint64_t m = (mask >> half) & 0xFFFF;
- if (m == 0) {
- escape_store16(out, escape_load16(src + i + half));
- out += 16;
- } else {
- out = escape_block(src, out, i + half, i + half + 16, m);
- }
- }
- }
- i += 32;
- }
-#endif
- while (i + 16 <= len) {
- escape_vector word = escape_load16(src + i);
- escape_vector flags = escape_flags(word);
- if (simdjson_likely(!escape_any(flags))) {
- escape_store16(out, word);
- out += 16;
- } else {
- out = escape_block(src, out, i, i + 16, escape_bitmask(flags));
- }
- i += 16;
- }
- if (i < len) {
- const size_t rem = len - i;
- uint64_t m;
- if (len >= 16) {
- // The last 16 bytes of the input are in bounds. Bit k of that block's
- // mask belongs to input position len - 16 + k, so shift it down to align
- // bit 0 with position i.
- m = escape_mask(escape_load16(src + len - 16)) >>
- (escape_mask_bits * (16 - rem));
- } else if (len >= 8) {
- // Here i == 0 and rem == len. Two overlapping 8-byte loads cover
- // [0, 8) and [len - 8, len), which is the whole input since len < 16.
- uint64_t mm = escape_mask(escape_load8x2(src, src + len - 8));
- constexpr uint64_t low8 = (uint64_t(1) << (escape_mask_bits * 8)) - 1;
- m = (mm & low8) |
- ((mm >> (escape_mask_bits * 8)) << (escape_mask_bits * (len - 8)));
- } else if (len >= 4) {
- // Same idea with two overlapping 4-byte loads.
- uint64_t mm = escape_mask(escape_load4x2(src, src + len - 4));
- constexpr uint64_t low4 = (uint64_t(1) << (escape_mask_bits * 4)) - 1;
- m = (mm & low4) | (((mm >> (escape_mask_bits * 4)) & low4)
- << (escape_mask_bits * (len - 4)));
- } else {
- // Fewer than 4 bytes: at most three table lookups, no need for SIMD.
- for (size_t k = 0; k < len; k++) {
- uint8_t c = src[k];
- if (json_quotable_character[c]) {
- escape_json_char(char(c), out);
- } else {
- *out++ = char(c);
- }
- }
- return size_t(out - initout);
- }
- if (m == 0) {
- copy_lt16(out, src + i, rem);
- out += rem;
- } else {
- out = escape_block(src, out, i, len, m);
- }
- }
- return size_t(out - initout);
-}
-
-#else // SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-
-// Writes the escaped version of input to out, returning the number of bytes
-// written. Uses SIMD position finding to locate quotable characters efficiently.
-inline size_t write_string_escaped(const std::string_view input, char *out) {
- size_t mysize = input.size();
-
- // Use SIMD position finder directly - it returns mysize if no escape needed
- size_t location = find_next_json_quotable_character(input, 0);
- if (location == mysize) {
- // Fast path: no escaping needed
- memcpy(out, input.data(), input.size());
- return input.size();
- }
-
- const char *const initout = out;
- memcpy(out, input.data(), location);
- out += location;
- escape_json_char(input[location], out);
- location += 1;
- while (location < mysize) {
- size_t newlocation = find_next_json_quotable_character(input, location);
- memcpy(out, input.data() + location, newlocation - location);
- out += newlocation - location;
- location = newlocation;
- if (location == mysize) {
- break;
- }
- escape_json_char(input[location], out);
- location += 1;
- }
- return size_t(out - initout);
-}
-
-#endif // SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-#undef SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-
namespace internal {
// Sets the size of s to n without initializing the new bytes when the
@@ -51224,6 +50756,7 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin
/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */
/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_H */
/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */
+/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */
/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */
namespace simdjson {
@@ -51260,6 +50793,9 @@ struct has_custom_serialization : std::false_type {};
namespace haswell {
namespace builder {
+
+using simdjson::internal::escape_best::write_string_escaped;
+
/**
* A builder for JSON strings representing documents. This is a low-level
* builder that is not meant to be used directly by end-users. Though it
@@ -51606,6 +51142,7 @@ simdjson_warn_unused error_code to_json(const Z &z, std::string &s, size_t initi
/* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */
/* amalgamation skipped (editor-only): #include "simdjson/concepts.h" */
/* amalgamation skipped (editor-only): #include "simdjson/annotations.h" */
+/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */
/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */
#if SIMDJSON_STATIC_REFLECTION
@@ -51631,7 +51168,6 @@ namespace internal {
using simdjson::internal::write_uint_jeaiii;
simdjson_inline char *write_double(char *p, double v) noexcept;
} // namespace internal
-simdjson_really_inline size_t write_string_escaped(const std::string_view input, char *out);
SIMDJSON_PUSH_DISABLE_WARNINGS
SIMDJSON_DISABLE_GCC_WARNING(-Warray-bounds)
@@ -52652,619 +52188,13 @@ simdjson_warn_unused simdjson_result<std::string> extract_fractured_json(
/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_INL_H */
/* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */
/* amalgamation skipped (editor-only): #include "simdjson/internal/jeaiii.h" */
+/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */
/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */
-/*
- * Empirically, we have found that an inlined optimization is important for
- * performance. The following macros are not ideal. We should find a better
- * way to inline the code.
- */
-
-#if defined(__SSE2__) || defined(__x86_64__) || defined(__x86_64) || \
- (defined(_M_AMD64) || defined(_M_X64) || \
- (defined(_M_IX86_FP) && _M_IX86_FP == 2))
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_SSE2
-#define SIMDJSON_EXPERIMENTAL_HAS_SSE2 1
-#endif
-#endif
-
-#if defined(__aarch64__) || defined(_M_ARM64)
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_NEON
-#define SIMDJSON_EXPERIMENTAL_HAS_NEON 1
-#endif
-#endif
-#if defined(__loongarch_sx)
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_LSX
-#define SIMDJSON_EXPERIMENTAL_HAS_LSX 1
-#endif
-#endif
-#if defined(__loongarch_asx)
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_LASX
-#define SIMDJSON_EXPERIMENTAL_HAS_LASX 1
-#endif
-#endif
-#if defined(__riscv_v_intrinsic) && __riscv_v_intrinsic >= 11000 && \
- defined(__riscv_vector)
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_RVV
-#define SIMDJSON_EXPERIMENTAL_HAS_RVV 1
-#endif
-#endif
-#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) && defined(__POWER8_VECTOR__)
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64
-#define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1
-#endif
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_NEON
-#include <arm_neon.h>
-#ifdef _MSC_VER
-#include <intrin.h>
-#endif
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_SSE2
-#include <emmintrin.h>
-#if defined(__AVX2__)
-#include <immintrin.h>
-#endif
-#ifdef _MSC_VER
-#include <intrin.h>
-#endif
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_LSX
-#include <lsxintrin.h>
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_LASX
-#include <lasxintrin.h>
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_RVV
-#include <riscv_vector.h>
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_PPC64
-#include <altivec.h>
-#ifdef bool
-#undef bool
-#endif
-#ifdef vector
-#undef vector
-#endif
-#endif
-
-
namespace simdjson {
namespace haswell {
namespace builder {
-static SIMDJSON_CONSTEXPR_LAMBDA std::array<uint8_t, 256>
- json_quotable_character = {
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
-
-/**
-
-A possible SWAR implementation of has_json_escapable_byte. It is not used
-because it is slower than the current implementation. It is kept here for
-reference (to show that we tried it).
-
-inline bool has_json_escapable_byte(uint64_t x) {
- uint64_t is_ascii = 0x8080808080808080ULL & ~x;
- uint64_t xor2 = x ^ 0x0202020202020202ULL;
- uint64_t lt32_or_eq34 = xor2 - 0x2121212121212121ULL;
- uint64_t sub92 = x ^ 0x5C5C5C5C5C5C5C5CULL;
- uint64_t eq92 = (sub92 - 0x0101010101010101ULL);
- return ((lt32_or_eq34 | eq92) & is_ascii) != 0;
-}
-
-**/
-
-#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 || SIMDJSON_EXPERIMENTAL_HAS_NEON
-#define SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE 1
-#endif
-
-// With block escaping (SSE2/NEON), write_string_escaped scans the input itself.
-#if !SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-// Scalar fallback for finding next quotable character
-SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline size_t
-find_next_json_quotable_character_scalar(const std::string_view view,
- size_t location) noexcept {
- for (auto pos = view.begin() + location; pos != view.end(); ++pos) {
- if (json_quotable_character[static_cast<uint8_t>(*pos)]) {
- return pos - view.begin();
- }
- }
- return size_t(view.size());
-}
-
-// SIMD-accelerated position finding that directly locates the first quotable
-// character, combining detection and position extraction in a single pass to
-// minimize redundant work.
-#if SIMDJSON_EXPERIMENTAL_HAS_LASX
-simdjson_inline size_t
-find_next_json_quotable_character(const std::string_view view,
- size_t location) noexcept {
- const size_t len = view.size();
- const uint8_t *ptr =
- reinterpret_cast<const uint8_t *>(view.data()) + location;
- size_t remaining = len - location;
-
- // SIMD constants for characters requiring escape
- __m256i v34 = __lasx_xvreplgr2vr_b(34); // '"'
- __m256i v92 = __lasx_xvreplgr2vr_b(92); // '\\'
- __m256i v32 = __lasx_xvreplgr2vr_b(32); // control char threshold
-
- while (remaining >= 32) {
- __m256i word = __lasx_xvld(ptr, 0);
-
- // Check for quotable characters: '"', '\\', or control chars (< 32)
- __m256i needs_escape = __lasx_xvseq_b(word, v34);
- needs_escape = __lasx_xvor_v(needs_escape, __lasx_xvseq_b(word, v92));
- needs_escape = __lasx_xvor_v(needs_escape, __lasx_xvslt_bu(word, v32));
-
- if (!__lasx_xbz_v(needs_escape)) {
- // Found a quotable character - locate it via the four 64-bit lanes
- uint64_t lane0 = __lasx_xvpickve2gr_du(needs_escape, 0);
- uint64_t lane1 = __lasx_xvpickve2gr_du(needs_escape, 1);
- uint64_t lane2 = __lasx_xvpickve2gr_du(needs_escape, 2);
- uint64_t lane3 = __lasx_xvpickve2gr_du(needs_escape, 3);
- size_t offset = ptr - reinterpret_cast<const uint8_t *>(view.data());
- if (lane0 != 0) {
- return offset + trailing_zeroes(lane0) / 8;
- } else if (lane1 != 0) {
- return offset + 8 + trailing_zeroes(lane1) / 8;
- } else if (lane2 != 0) {
- return offset + 16 + trailing_zeroes(lane2) / 8;
- } else {
- return offset + 24 + trailing_zeroes(lane3) / 8;
- }
- }
- ptr += 32;
- remaining -= 32;
- }
- size_t current = len - remaining;
- return find_next_json_quotable_character_scalar(view, current);
-}
-#elif SIMDJSON_EXPERIMENTAL_HAS_LSX
-simdjson_inline size_t
-find_next_json_quotable_character(const std::string_view view,
- size_t location) noexcept {
- const size_t len = view.size();
- const uint8_t *ptr =
- reinterpret_cast<const uint8_t *>(view.data()) + location;
- size_t remaining = len - location;
-
- //SIMD constants for characters requiring escape
- __m128i v34 = __lsx_vreplgr2vr_b(34); // '"'
- __m128i v92 = __lsx_vreplgr2vr_b(92); // '\\'
- __m128i v32 = __lsx_vreplgr2vr_b(32); // control char threshold
-
- while (remaining >= 16){
- __m128i word = __lsx_vld(ptr, 0);
-
- //Check for the quotable characters: '"', '\\', or control char (<32)
- __m128i needs_escape = __lsx_vseq_b(word, v34);
- needs_escape = __lsx_vor_v(needs_escape, __lsx_vseq_b(word, v92));
- needs_escape = __lsx_vor_v(needs_escape, __lsx_vslt_bu(word, v32));
-
- if (!__lsx_bz_v(needs_escape)){
-
- //Found quotable character - extract exact byte position
- uint64_t lo = __lsx_vpickve2gr_du(needs_escape,0);
- uint64_t hi = __lsx_vpickve2gr_du(needs_escape,1);
- size_t offset = ptr - reinterpret_cast<const uint8_t *>(view.data());
- if ( lo != 0) {
- return offset + trailing_zeroes(lo) / 8;
- } else {
- return offset + 8 + trailing_zeroes(hi) / 8;
- }
- }
- ptr += 16;
- remaining -= 16;
- }
- size_t current = len - remaining;
- return find_next_json_quotable_character_scalar(view, current);
-}
-#elif SIMDJSON_EXPERIMENTAL_HAS_RVV
-simdjson_inline size_t
-find_next_json_quotable_character(const std::string_view view,
- size_t location) noexcept {
- const size_t len = view.size();
- const uint8_t *ptr =
- reinterpret_cast<const uint8_t *>(view.data()) + location;
- size_t remaining = len - location;
-
- while (remaining > 0) {
- size_t vl = __riscv_vsetvl_e8m1(remaining);
- vuint8m1_t word = __riscv_vle8_v_u8m1(ptr, vl);
-
- // Check for quotable characters: '"', '\\', or control chars (< 32)
- vbool8_t needs_escape = __riscv_vmseq(word, (uint8_t)34, vl);
- needs_escape = __riscv_vmor(needs_escape,
- __riscv_vmseq(word, (uint8_t)92, vl), vl);
- needs_escape = __riscv_vmor(needs_escape,
- __riscv_vmsltu(word, (uint8_t)32, vl), vl);
-
- long first = __riscv_vfirst(needs_escape, vl);
- if (first >= 0) {
- size_t offset = ptr - reinterpret_cast<const uint8_t *>(view.data());
- return offset + first;
- }
- ptr += vl;
- remaining -= vl;
- }
-
- return len;
-}
-#elif SIMDJSON_EXPERIMENTAL_HAS_PPC64
-simdjson_inline size_t
-find_next_json_quotable_character(const std::string_view view,
- size_t location) noexcept {
- const size_t len = view.size();
- const uint8_t *ptr =
- reinterpret_cast<const uint8_t *>(view.data()) + location;
- size_t remaining = len - location;
-
- // SIMD constants for characters requiring escape
- __vector unsigned char v34 = vec_splats((unsigned char)34); // '"'
- __vector unsigned char v92 = vec_splats((unsigned char)92); // '\\'
- __vector unsigned char v32 = vec_splats((unsigned char)32); // control char threshold
-
- // Bitmask for vec_vbpermq to extract one bit per byte
- const __vector unsigned char perm_mask = {0x78, 0x70, 0x68, 0x60, 0x58, 0x50,
- 0x48, 0x40, 0x38, 0x30, 0x28, 0x20,
- 0x18, 0x10, 0x08, 0x00};
-
- while (remaining >= 16) {
- __vector unsigned char word =
- vec_vsx_ld(0, reinterpret_cast<const unsigned char *>(ptr));
-
- // Check for quotable characters: '"', '\\', or control chars (< 32)
- __vector unsigned char needs_escape =
- (__vector unsigned char)vec_cmpeq(word, v34);
- needs_escape = vec_or(needs_escape,
- (__vector unsigned char)vec_cmpeq(word, v92));
- needs_escape = vec_or(needs_escape,
- (__vector unsigned char)vec_cmplt(word, v32));
-
- __vector unsigned long long result =
- (__vector unsigned long long)vec_vbpermq(needs_escape, perm_mask);
-#ifdef __LITTLE_ENDIAN__
- unsigned int mask = static_cast<unsigned int>(result[1]);
-#else
- unsigned int mask = static_cast<unsigned int>(result[0]);
-#endif
- if (mask != 0) {
- size_t offset = ptr - reinterpret_cast<const uint8_t *>(view.data());
- return offset + __builtin_ctz(mask);
- }
- ptr += 16;
- remaining -= 16;
- }
-
- // Scalar fallback for remaining bytes
- size_t current = len - remaining;
- return find_next_json_quotable_character_scalar(view, current);
-}
-#else
-SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline size_t
-find_next_json_quotable_character(const std::string_view view,
- size_t location) noexcept {
- return find_next_json_quotable_character_scalar(view, location);
-}
-#endif
-#endif // !SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-
-SIMDJSON_CONSTEXPR_LAMBDA static std::string_view control_chars[] = {
- "\\u0000", "\\u0001", "\\u0002", "\\u0003", "\\u0004", "\\u0005", "\\u0006",
- "\\u0007", "\\b", "\\t", "\\n", "\\u000b", "\\f", "\\r",
- "\\u000e", "\\u000f", "\\u0010", "\\u0011", "\\u0012", "\\u0013", "\\u0014",
- "\\u0015", "\\u0016", "\\u0017", "\\u0018", "\\u0019", "\\u001a", "\\u001b",
- "\\u001c", "\\u001d", "\\u001e", "\\u001f"};
-
-// All Unicode characters may be placed within the quotation marks, except for
-// the characters that MUST be escaped: quotation mark, reverse solidus, and the
-// control characters (U+0000 through U+001F). There are two-character sequence
-// escape representations of some popular characters:
-// \", \\, \b, \f, \n, \r, \t.
-SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline void escape_json_char(char c, char *&out) {
- if (c == '"') {
- memcpy(out, "\\\"", 2);
- out += 2;
- } else if (c == '\\') {
- memcpy(out, "\\\\", 2);
- out += 2;
- } else {
- std::string_view v = control_chars[uint8_t(c)];
- memcpy(out, v.data(), v.size());
- out += v.size();
- }
-}
-
-#if SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-
-#if SIMDJSON_EXPERIMENTAL_HAS_SSE2
-
-using escape_vector = __m128i;
-// Mask bits that each input byte contributes to escape_bitmask().
-static constexpr unsigned escape_mask_bits = 1;
-
-simdjson_inline escape_vector escape_load16(const uint8_t *p) noexcept {
- return _mm_loadu_si128(reinterpret_cast<const __m128i *>(p));
-}
-
-simdjson_inline void escape_store16(char *out, escape_vector v) noexcept {
- _mm_storeu_si128(reinterpret_cast<__m128i *>(out), v);
-}
-
-// Builds a vector whose bytes 0..7 come from a and bytes 8..15 from b.
-simdjson_inline escape_vector escape_load8x2(const uint8_t *a,
- const uint8_t *b) noexcept {
- return _mm_unpacklo_epi64(
- _mm_loadl_epi64(reinterpret_cast<const __m128i *>(a)),
- _mm_loadl_epi64(reinterpret_cast<const __m128i *>(b)));
-}
-
-// Builds a vector whose bytes 0..3 come from a and bytes 4..7 from b. The
-// upper half is unspecified; callers only look at the low eight bits.
-simdjson_inline escape_vector escape_load4x2(const uint8_t *a,
- const uint8_t *b) noexcept {
- int32_t a32, b32;
- memcpy(&a32, a, 4);
- memcpy(&b32, b, 4);
- return _mm_unpacklo_epi32(_mm_cvtsi32_si128(a32), _mm_cvtsi32_si128(b32));
-}
-
-// Sets every bit of byte k when byte k of v is a quotable character ('"',
-// '\\' or a control character).
-simdjson_inline escape_vector escape_flags(escape_vector v) noexcept {
- const __m128i v34 = _mm_set1_epi8(34); // '"'
- const __m128i v92 = _mm_set1_epi8(92); // '\\'
- const __m128i v31 = _mm_set1_epi8(31); // for control char detection
- __m128i needs_escape = _mm_cmpeq_epi8(v, v34);
- needs_escape = _mm_or_si128(needs_escape, _mm_cmpeq_epi8(v, v92));
- return _mm_or_si128(
- needs_escape, _mm_cmpeq_epi8(_mm_subs_epu8(v, v31), _mm_setzero_si128()));
-}
-
-// True when any byte needs escaping. Kept separate from escape_bitmask
-// because some instruction sets can answer it without leaving the vector
-// register file; on SSE2 the compiler folds the two together.
-simdjson_inline bool escape_any(escape_vector flags) noexcept {
- return _mm_movemask_epi8(flags) != 0;
-}
-
-// A 16-bit mask with bit k set when byte k needed escaping.
-simdjson_inline uint64_t escape_bitmask(escape_vector flags) noexcept {
- return uint64_t(uint32_t(_mm_movemask_epi8(flags)));
-}
-
-#else // SIMDJSON_EXPERIMENTAL_HAS_NEON
-
-using escape_vector = uint8x16_t;
-static constexpr unsigned escape_mask_bits = 4;
-
-simdjson_inline escape_vector escape_load16(const uint8_t *p) noexcept {
- return vld1q_u8(p);
-}
-
-simdjson_inline void escape_store16(char *out, escape_vector v) noexcept {
- vst1q_u8(reinterpret_cast<uint8_t *>(out), v);
-}
-
-simdjson_inline escape_vector escape_load8x2(const uint8_t *a,
- const uint8_t *b) noexcept {
- uint64_t a64, b64;
- memcpy(&a64, a, 8);
- memcpy(&b64, b, 8);
- return vreinterpretq_u8_u64(vsetq_lane_u64(b64, vdupq_n_u64(a64), 1));
-}
-
-simdjson_inline escape_vector escape_load4x2(const uint8_t *a,
- const uint8_t *b) noexcept {
- uint32_t a32, b32;
- memcpy(&a32, a, 4);
- memcpy(&b32, b, 4);
- return vreinterpretq_u8_u32(vsetq_lane_u32(b32, vdupq_n_u32(a32), 1));
-}
-
-simdjson_inline escape_vector escape_flags(escape_vector v) noexcept {
- uint8x16_t needs_escape = vceqq_u8(v, vdupq_n_u8(34)); // '"'
- needs_escape = vorrq_u8(needs_escape, vceqq_u8(v, vdupq_n_u8(92))); // '\\'
- return vorrq_u8(needs_escape, vcltq_u8(v, vdupq_n_u8(32)));
-}
-
-simdjson_inline bool escape_any(escape_vector flags) noexcept {
- return vmaxvq_u32(vreinterpretq_u32_u8(flags)) != 0;
-}
-
-// Four bits per byte rather than one: escape_block and the tail paths scale
-// their shifts by escape_mask_bits to match.
-simdjson_inline uint64_t escape_bitmask(escape_vector flags) noexcept {
- uint8x8_t narrowed = vshrn_n_u16(vreinterpretq_u16_u8(flags), 4);
- return vget_lane_u64(vreinterpret_u64_u8(narrowed), 0);
-}
-
-#endif // instruction set selection
-
-// The escape bitmask of a 16-byte block.
-simdjson_inline uint64_t escape_mask(escape_vector v) noexcept {
- return escape_bitmask(escape_flags(v));
-}
-
-// Copies n bytes with n < 16, using overlapping loads and stores. It never
-// reads more than n bytes from src, nor writes more than n bytes to dst.
-simdjson_inline void copy_lt16(char *dst, const uint8_t *src,
- size_t n) noexcept {
- if (n >= 8) {
- memcpy(dst, src, 8);
- memcpy(dst + n - 8, src + n - 8, 8);
- } else if (n >= 4) {
- memcpy(dst, src, 4);
- memcpy(dst + n - 4, src + n - 4, 4);
- } else if (n > 0) {
- dst[0] = char(src[0]);
- dst[n >> 1] = char(src[n >> 1]);
- dst[n - 1] = char(src[n - 1]);
- }
-}
-
-// Escapes the bytes of src in the range [i, blockend), given that m is the
-// (non-zero) escape mask for that range: the escape_mask_bits-wide lane of m
-// at byte k is non-zero when src[i + k] requires escaping. Returns the updated
-// output pointer.
-simdjson_never_inline char *escape_block(const uint8_t *src, char *out,
- size_t i, size_t blockend,
- uint64_t m) noexcept {
- constexpr uint64_t lane = (uint64_t(1) << escape_mask_bits) - 1;
-
- size_t pos = i; // first byte not yet copied
- while (m) {
- const size_t tz = trailing_zeroes(m);
- const size_t next = i + tz / escape_mask_bits;
- // Copy the run of safe bytes that precedes this escape.
- copy_lt16(out, src + pos, next - pos);
- out += next - pos;
- escape_json_char(char(src[next]), out);
- pos = next + 1;
- m &= ~(lane << tz);
- }
- // Copy whatever follows the last escape.
- copy_lt16(out, src + pos, blockend - pos);
- return out + (blockend - pos);
-}
-
-// Writes the escaped version of input to out, returning the number of bytes
-// written.
-simdjson_really_inline size_t write_string_escaped(const std::string_view input, char *out) {
- const size_t len = input.size();
- const uint8_t *src = reinterpret_cast<const uint8_t *>(input.data());
- const char *const initout = out;
-
- size_t i = 0;
-#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 && defined(__AVX2__)
- while (i + 32 <= len) {
- const __m256i word = _mm256_loadu_si256(reinterpret_cast<const __m256i *>(src + i));
- const __m256i flags = _mm256_or_si256(
- _mm256_or_si256(_mm256_cmpeq_epi8(word, _mm256_set1_epi8(34)), // '"'
- _mm256_cmpeq_epi8(word, _mm256_set1_epi8(92))), // '\\'
- _mm256_cmpeq_epi8(_mm256_subs_epu8(word, _mm256_set1_epi8(31)),
- _mm256_setzero_si256())); // control
- const uint32_t mask = uint32_t(_mm256_movemask_epi8(flags));
- if (simdjson_likely(mask == 0)) {
- _mm256_storeu_si256(reinterpret_cast<__m256i *>(out), word);
- out += 32;
- } else {
- for (size_t half = 0; half < 32; half += 16) {
- const uint64_t m = (mask >> half) & 0xFFFF;
- if (m == 0) {
- escape_store16(out, escape_load16(src + i + half));
- out += 16;
- } else {
- out = escape_block(src, out, i + half, i + half + 16, m);
- }
- }
- }
- i += 32;
- }
-#endif
- while (i + 16 <= len) {
- escape_vector word = escape_load16(src + i);
- escape_vector flags = escape_flags(word);
- if (simdjson_likely(!escape_any(flags))) {
- escape_store16(out, word);
- out += 16;
- } else {
- out = escape_block(src, out, i, i + 16, escape_bitmask(flags));
- }
- i += 16;
- }
- if (i < len) {
- const size_t rem = len - i;
- uint64_t m;
- if (len >= 16) {
- // The last 16 bytes of the input are in bounds. Bit k of that block's
- // mask belongs to input position len - 16 + k, so shift it down to align
- // bit 0 with position i.
- m = escape_mask(escape_load16(src + len - 16)) >>
- (escape_mask_bits * (16 - rem));
- } else if (len >= 8) {
- // Here i == 0 and rem == len. Two overlapping 8-byte loads cover
- // [0, 8) and [len - 8, len), which is the whole input since len < 16.
- uint64_t mm = escape_mask(escape_load8x2(src, src + len - 8));
- constexpr uint64_t low8 = (uint64_t(1) << (escape_mask_bits * 8)) - 1;
- m = (mm & low8) |
- ((mm >> (escape_mask_bits * 8)) << (escape_mask_bits * (len - 8)));
- } else if (len >= 4) {
- // Same idea with two overlapping 4-byte loads.
- uint64_t mm = escape_mask(escape_load4x2(src, src + len - 4));
- constexpr uint64_t low4 = (uint64_t(1) << (escape_mask_bits * 4)) - 1;
- m = (mm & low4) | (((mm >> (escape_mask_bits * 4)) & low4)
- << (escape_mask_bits * (len - 4)));
- } else {
- // Fewer than 4 bytes: at most three table lookups, no need for SIMD.
- for (size_t k = 0; k < len; k++) {
- uint8_t c = src[k];
- if (json_quotable_character[c]) {
- escape_json_char(char(c), out);
- } else {
- *out++ = char(c);
- }
- }
- return size_t(out - initout);
- }
- if (m == 0) {
- copy_lt16(out, src + i, rem);
- out += rem;
- } else {
- out = escape_block(src, out, i, len, m);
- }
- }
- return size_t(out - initout);
-}
-
-#else // SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-
-// Writes the escaped version of input to out, returning the number of bytes
-// written. Uses SIMD position finding to locate quotable characters efficiently.
-inline size_t write_string_escaped(const std::string_view input, char *out) {
- size_t mysize = input.size();
-
- // Use SIMD position finder directly - it returns mysize if no escape needed
- size_t location = find_next_json_quotable_character(input, 0);
- if (location == mysize) {
- // Fast path: no escaping needed
- memcpy(out, input.data(), input.size());
- return input.size();
- }
-
- const char *const initout = out;
- memcpy(out, input.data(), location);
- out += location;
- escape_json_char(input[location], out);
- location += 1;
- while (location < mysize) {
- size_t newlocation = find_next_json_quotable_character(input, location);
- memcpy(out, input.data() + location, newlocation - location);
- out += newlocation - location;
- location = newlocation;
- if (location == mysize) {
- break;
- }
- escape_json_char(input[location], out);
- location += 1;
- }
- return size_t(out - initout);
-}
-
-#endif // SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-#undef SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-
namespace internal {
// Sets the size of s to n without initializing the new bytes when the
@@ -54585,6 +53515,7 @@ simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t
/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */
/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_H */
/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */
+/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */
/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */
namespace simdjson {
@@ -54621,6 +53552,9 @@ struct has_custom_serialization : std::false_type {};
namespace icelake {
namespace builder {
+
+using simdjson::internal::escape_best::write_string_escaped;
+
/**
* A builder for JSON strings representing documents. This is a low-level
* builder that is not meant to be used directly by end-users. Though it
@@ -54967,6 +53901,7 @@ simdjson_warn_unused error_code to_json(const Z &z, std::string &s, size_t initi
/* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */
/* amalgamation skipped (editor-only): #include "simdjson/concepts.h" */
/* amalgamation skipped (editor-only): #include "simdjson/annotations.h" */
+/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */
/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */
#if SIMDJSON_STATIC_REFLECTION
@@ -54992,7 +53927,6 @@ namespace internal {
using simdjson::internal::write_uint_jeaiii;
simdjson_inline char *write_double(char *p, double v) noexcept;
} // namespace internal
-simdjson_really_inline size_t write_string_escaped(const std::string_view input, char *out);
SIMDJSON_PUSH_DISABLE_WARNINGS
SIMDJSON_DISABLE_GCC_WARNING(-Warray-bounds)
@@ -56013,619 +54947,13 @@ simdjson_warn_unused simdjson_result<std::string> extract_fractured_json(
/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_INL_H */
/* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */
/* amalgamation skipped (editor-only): #include "simdjson/internal/jeaiii.h" */
+/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */
/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */
-/*
- * Empirically, we have found that an inlined optimization is important for
- * performance. The following macros are not ideal. We should find a better
- * way to inline the code.
- */
-
-#if defined(__SSE2__) || defined(__x86_64__) || defined(__x86_64) || \
- (defined(_M_AMD64) || defined(_M_X64) || \
- (defined(_M_IX86_FP) && _M_IX86_FP == 2))
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_SSE2
-#define SIMDJSON_EXPERIMENTAL_HAS_SSE2 1
-#endif
-#endif
-
-#if defined(__aarch64__) || defined(_M_ARM64)
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_NEON
-#define SIMDJSON_EXPERIMENTAL_HAS_NEON 1
-#endif
-#endif
-#if defined(__loongarch_sx)
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_LSX
-#define SIMDJSON_EXPERIMENTAL_HAS_LSX 1
-#endif
-#endif
-#if defined(__loongarch_asx)
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_LASX
-#define SIMDJSON_EXPERIMENTAL_HAS_LASX 1
-#endif
-#endif
-#if defined(__riscv_v_intrinsic) && __riscv_v_intrinsic >= 11000 && \
- defined(__riscv_vector)
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_RVV
-#define SIMDJSON_EXPERIMENTAL_HAS_RVV 1
-#endif
-#endif
-#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) && defined(__POWER8_VECTOR__)
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64
-#define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1
-#endif
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_NEON
-#include <arm_neon.h>
-#ifdef _MSC_VER
-#include <intrin.h>
-#endif
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_SSE2
-#include <emmintrin.h>
-#if defined(__AVX2__)
-#include <immintrin.h>
-#endif
-#ifdef _MSC_VER
-#include <intrin.h>
-#endif
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_LSX
-#include <lsxintrin.h>
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_LASX
-#include <lasxintrin.h>
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_RVV
-#include <riscv_vector.h>
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_PPC64
-#include <altivec.h>
-#ifdef bool
-#undef bool
-#endif
-#ifdef vector
-#undef vector
-#endif
-#endif
-
-
namespace simdjson {
namespace icelake {
namespace builder {
-static SIMDJSON_CONSTEXPR_LAMBDA std::array<uint8_t, 256>
- json_quotable_character = {
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
-
-/**
-
-A possible SWAR implementation of has_json_escapable_byte. It is not used
-because it is slower than the current implementation. It is kept here for
-reference (to show that we tried it).
-
-inline bool has_json_escapable_byte(uint64_t x) {
- uint64_t is_ascii = 0x8080808080808080ULL & ~x;
- uint64_t xor2 = x ^ 0x0202020202020202ULL;
- uint64_t lt32_or_eq34 = xor2 - 0x2121212121212121ULL;
- uint64_t sub92 = x ^ 0x5C5C5C5C5C5C5C5CULL;
- uint64_t eq92 = (sub92 - 0x0101010101010101ULL);
- return ((lt32_or_eq34 | eq92) & is_ascii) != 0;
-}
-
-**/
-
-#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 || SIMDJSON_EXPERIMENTAL_HAS_NEON
-#define SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE 1
-#endif
-
-// With block escaping (SSE2/NEON), write_string_escaped scans the input itself.
-#if !SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-// Scalar fallback for finding next quotable character
-SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline size_t
-find_next_json_quotable_character_scalar(const std::string_view view,
- size_t location) noexcept {
- for (auto pos = view.begin() + location; pos != view.end(); ++pos) {
- if (json_quotable_character[static_cast<uint8_t>(*pos)]) {
- return pos - view.begin();
- }
- }
- return size_t(view.size());
-}
-
-// SIMD-accelerated position finding that directly locates the first quotable
-// character, combining detection and position extraction in a single pass to
-// minimize redundant work.
-#if SIMDJSON_EXPERIMENTAL_HAS_LASX
-simdjson_inline size_t
-find_next_json_quotable_character(const std::string_view view,
- size_t location) noexcept {
- const size_t len = view.size();
- const uint8_t *ptr =
- reinterpret_cast<const uint8_t *>(view.data()) + location;
- size_t remaining = len - location;
-
- // SIMD constants for characters requiring escape
- __m256i v34 = __lasx_xvreplgr2vr_b(34); // '"'
- __m256i v92 = __lasx_xvreplgr2vr_b(92); // '\\'
- __m256i v32 = __lasx_xvreplgr2vr_b(32); // control char threshold
-
- while (remaining >= 32) {
- __m256i word = __lasx_xvld(ptr, 0);
-
- // Check for quotable characters: '"', '\\', or control chars (< 32)
- __m256i needs_escape = __lasx_xvseq_b(word, v34);
- needs_escape = __lasx_xvor_v(needs_escape, __lasx_xvseq_b(word, v92));
- needs_escape = __lasx_xvor_v(needs_escape, __lasx_xvslt_bu(word, v32));
-
- if (!__lasx_xbz_v(needs_escape)) {
- // Found a quotable character - locate it via the four 64-bit lanes
- uint64_t lane0 = __lasx_xvpickve2gr_du(needs_escape, 0);
- uint64_t lane1 = __lasx_xvpickve2gr_du(needs_escape, 1);
- uint64_t lane2 = __lasx_xvpickve2gr_du(needs_escape, 2);
- uint64_t lane3 = __lasx_xvpickve2gr_du(needs_escape, 3);
- size_t offset = ptr - reinterpret_cast<const uint8_t *>(view.data());
- if (lane0 != 0) {
- return offset + trailing_zeroes(lane0) / 8;
- } else if (lane1 != 0) {
- return offset + 8 + trailing_zeroes(lane1) / 8;
- } else if (lane2 != 0) {
- return offset + 16 + trailing_zeroes(lane2) / 8;
- } else {
- return offset + 24 + trailing_zeroes(lane3) / 8;
- }
- }
- ptr += 32;
- remaining -= 32;
- }
- size_t current = len - remaining;
- return find_next_json_quotable_character_scalar(view, current);
-}
-#elif SIMDJSON_EXPERIMENTAL_HAS_LSX
-simdjson_inline size_t
-find_next_json_quotable_character(const std::string_view view,
- size_t location) noexcept {
- const size_t len = view.size();
- const uint8_t *ptr =
- reinterpret_cast<const uint8_t *>(view.data()) + location;
- size_t remaining = len - location;
-
- //SIMD constants for characters requiring escape
- __m128i v34 = __lsx_vreplgr2vr_b(34); // '"'
- __m128i v92 = __lsx_vreplgr2vr_b(92); // '\\'
- __m128i v32 = __lsx_vreplgr2vr_b(32); // control char threshold
-
- while (remaining >= 16){
- __m128i word = __lsx_vld(ptr, 0);
-
- //Check for the quotable characters: '"', '\\', or control char (<32)
- __m128i needs_escape = __lsx_vseq_b(word, v34);
- needs_escape = __lsx_vor_v(needs_escape, __lsx_vseq_b(word, v92));
- needs_escape = __lsx_vor_v(needs_escape, __lsx_vslt_bu(word, v32));
-
- if (!__lsx_bz_v(needs_escape)){
-
- //Found quotable character - extract exact byte position
- uint64_t lo = __lsx_vpickve2gr_du(needs_escape,0);
- uint64_t hi = __lsx_vpickve2gr_du(needs_escape,1);
- size_t offset = ptr - reinterpret_cast<const uint8_t *>(view.data());
- if ( lo != 0) {
- return offset + trailing_zeroes(lo) / 8;
- } else {
- return offset + 8 + trailing_zeroes(hi) / 8;
- }
- }
- ptr += 16;
- remaining -= 16;
- }
- size_t current = len - remaining;
- return find_next_json_quotable_character_scalar(view, current);
-}
-#elif SIMDJSON_EXPERIMENTAL_HAS_RVV
-simdjson_inline size_t
-find_next_json_quotable_character(const std::string_view view,
- size_t location) noexcept {
- const size_t len = view.size();
- const uint8_t *ptr =
- reinterpret_cast<const uint8_t *>(view.data()) + location;
- size_t remaining = len - location;
-
- while (remaining > 0) {
- size_t vl = __riscv_vsetvl_e8m1(remaining);
- vuint8m1_t word = __riscv_vle8_v_u8m1(ptr, vl);
-
- // Check for quotable characters: '"', '\\', or control chars (< 32)
- vbool8_t needs_escape = __riscv_vmseq(word, (uint8_t)34, vl);
- needs_escape = __riscv_vmor(needs_escape,
- __riscv_vmseq(word, (uint8_t)92, vl), vl);
- needs_escape = __riscv_vmor(needs_escape,
- __riscv_vmsltu(word, (uint8_t)32, vl), vl);
-
- long first = __riscv_vfirst(needs_escape, vl);
- if (first >= 0) {
- size_t offset = ptr - reinterpret_cast<const uint8_t *>(view.data());
- return offset + first;
- }
- ptr += vl;
- remaining -= vl;
- }
-
- return len;
-}
-#elif SIMDJSON_EXPERIMENTAL_HAS_PPC64
-simdjson_inline size_t
-find_next_json_quotable_character(const std::string_view view,
- size_t location) noexcept {
- const size_t len = view.size();
- const uint8_t *ptr =
- reinterpret_cast<const uint8_t *>(view.data()) + location;
- size_t remaining = len - location;
-
- // SIMD constants for characters requiring escape
- __vector unsigned char v34 = vec_splats((unsigned char)34); // '"'
- __vector unsigned char v92 = vec_splats((unsigned char)92); // '\\'
- __vector unsigned char v32 = vec_splats((unsigned char)32); // control char threshold
-
- // Bitmask for vec_vbpermq to extract one bit per byte
- const __vector unsigned char perm_mask = {0x78, 0x70, 0x68, 0x60, 0x58, 0x50,
- 0x48, 0x40, 0x38, 0x30, 0x28, 0x20,
- 0x18, 0x10, 0x08, 0x00};
-
- while (remaining >= 16) {
- __vector unsigned char word =
- vec_vsx_ld(0, reinterpret_cast<const unsigned char *>(ptr));
-
- // Check for quotable characters: '"', '\\', or control chars (< 32)
- __vector unsigned char needs_escape =
- (__vector unsigned char)vec_cmpeq(word, v34);
- needs_escape = vec_or(needs_escape,
- (__vector unsigned char)vec_cmpeq(word, v92));
- needs_escape = vec_or(needs_escape,
- (__vector unsigned char)vec_cmplt(word, v32));
-
- __vector unsigned long long result =
- (__vector unsigned long long)vec_vbpermq(needs_escape, perm_mask);
-#ifdef __LITTLE_ENDIAN__
- unsigned int mask = static_cast<unsigned int>(result[1]);
-#else
- unsigned int mask = static_cast<unsigned int>(result[0]);
-#endif
- if (mask != 0) {
- size_t offset = ptr - reinterpret_cast<const uint8_t *>(view.data());
- return offset + __builtin_ctz(mask);
- }
- ptr += 16;
- remaining -= 16;
- }
-
- // Scalar fallback for remaining bytes
- size_t current = len - remaining;
- return find_next_json_quotable_character_scalar(view, current);
-}
-#else
-SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline size_t
-find_next_json_quotable_character(const std::string_view view,
- size_t location) noexcept {
- return find_next_json_quotable_character_scalar(view, location);
-}
-#endif
-#endif // !SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-
-SIMDJSON_CONSTEXPR_LAMBDA static std::string_view control_chars[] = {
- "\\u0000", "\\u0001", "\\u0002", "\\u0003", "\\u0004", "\\u0005", "\\u0006",
- "\\u0007", "\\b", "\\t", "\\n", "\\u000b", "\\f", "\\r",
- "\\u000e", "\\u000f", "\\u0010", "\\u0011", "\\u0012", "\\u0013", "\\u0014",
- "\\u0015", "\\u0016", "\\u0017", "\\u0018", "\\u0019", "\\u001a", "\\u001b",
- "\\u001c", "\\u001d", "\\u001e", "\\u001f"};
-
-// All Unicode characters may be placed within the quotation marks, except for
-// the characters that MUST be escaped: quotation mark, reverse solidus, and the
-// control characters (U+0000 through U+001F). There are two-character sequence
-// escape representations of some popular characters:
-// \", \\, \b, \f, \n, \r, \t.
-SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline void escape_json_char(char c, char *&out) {
- if (c == '"') {
- memcpy(out, "\\\"", 2);
- out += 2;
- } else if (c == '\\') {
- memcpy(out, "\\\\", 2);
- out += 2;
- } else {
- std::string_view v = control_chars[uint8_t(c)];
- memcpy(out, v.data(), v.size());
- out += v.size();
- }
-}
-
-#if SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-
-#if SIMDJSON_EXPERIMENTAL_HAS_SSE2
-
-using escape_vector = __m128i;
-// Mask bits that each input byte contributes to escape_bitmask().
-static constexpr unsigned escape_mask_bits = 1;
-
-simdjson_inline escape_vector escape_load16(const uint8_t *p) noexcept {
- return _mm_loadu_si128(reinterpret_cast<const __m128i *>(p));
-}
-
-simdjson_inline void escape_store16(char *out, escape_vector v) noexcept {
- _mm_storeu_si128(reinterpret_cast<__m128i *>(out), v);
-}
-
-// Builds a vector whose bytes 0..7 come from a and bytes 8..15 from b.
-simdjson_inline escape_vector escape_load8x2(const uint8_t *a,
- const uint8_t *b) noexcept {
- return _mm_unpacklo_epi64(
- _mm_loadl_epi64(reinterpret_cast<const __m128i *>(a)),
- _mm_loadl_epi64(reinterpret_cast<const __m128i *>(b)));
-}
-
-// Builds a vector whose bytes 0..3 come from a and bytes 4..7 from b. The
-// upper half is unspecified; callers only look at the low eight bits.
-simdjson_inline escape_vector escape_load4x2(const uint8_t *a,
- const uint8_t *b) noexcept {
- int32_t a32, b32;
- memcpy(&a32, a, 4);
- memcpy(&b32, b, 4);
- return _mm_unpacklo_epi32(_mm_cvtsi32_si128(a32), _mm_cvtsi32_si128(b32));
-}
-
-// Sets every bit of byte k when byte k of v is a quotable character ('"',
-// '\\' or a control character).
-simdjson_inline escape_vector escape_flags(escape_vector v) noexcept {
- const __m128i v34 = _mm_set1_epi8(34); // '"'
- const __m128i v92 = _mm_set1_epi8(92); // '\\'
- const __m128i v31 = _mm_set1_epi8(31); // for control char detection
- __m128i needs_escape = _mm_cmpeq_epi8(v, v34);
- needs_escape = _mm_or_si128(needs_escape, _mm_cmpeq_epi8(v, v92));
- return _mm_or_si128(
- needs_escape, _mm_cmpeq_epi8(_mm_subs_epu8(v, v31), _mm_setzero_si128()));
-}
-
-// True when any byte needs escaping. Kept separate from escape_bitmask
-// because some instruction sets can answer it without leaving the vector
-// register file; on SSE2 the compiler folds the two together.
-simdjson_inline bool escape_any(escape_vector flags) noexcept {
- return _mm_movemask_epi8(flags) != 0;
-}
-
-// A 16-bit mask with bit k set when byte k needed escaping.
-simdjson_inline uint64_t escape_bitmask(escape_vector flags) noexcept {
- return uint64_t(uint32_t(_mm_movemask_epi8(flags)));
-}
-
-#else // SIMDJSON_EXPERIMENTAL_HAS_NEON
-
-using escape_vector = uint8x16_t;
-static constexpr unsigned escape_mask_bits = 4;
-
-simdjson_inline escape_vector escape_load16(const uint8_t *p) noexcept {
- return vld1q_u8(p);
-}
-
-simdjson_inline void escape_store16(char *out, escape_vector v) noexcept {
- vst1q_u8(reinterpret_cast<uint8_t *>(out), v);
-}
-
-simdjson_inline escape_vector escape_load8x2(const uint8_t *a,
- const uint8_t *b) noexcept {
- uint64_t a64, b64;
- memcpy(&a64, a, 8);
- memcpy(&b64, b, 8);
- return vreinterpretq_u8_u64(vsetq_lane_u64(b64, vdupq_n_u64(a64), 1));
-}
-
-simdjson_inline escape_vector escape_load4x2(const uint8_t *a,
- const uint8_t *b) noexcept {
- uint32_t a32, b32;
- memcpy(&a32, a, 4);
- memcpy(&b32, b, 4);
- return vreinterpretq_u8_u32(vsetq_lane_u32(b32, vdupq_n_u32(a32), 1));
-}
-
-simdjson_inline escape_vector escape_flags(escape_vector v) noexcept {
- uint8x16_t needs_escape = vceqq_u8(v, vdupq_n_u8(34)); // '"'
- needs_escape = vorrq_u8(needs_escape, vceqq_u8(v, vdupq_n_u8(92))); // '\\'
- return vorrq_u8(needs_escape, vcltq_u8(v, vdupq_n_u8(32)));
-}
-
-simdjson_inline bool escape_any(escape_vector flags) noexcept {
- return vmaxvq_u32(vreinterpretq_u32_u8(flags)) != 0;
-}
-
-// Four bits per byte rather than one: escape_block and the tail paths scale
-// their shifts by escape_mask_bits to match.
-simdjson_inline uint64_t escape_bitmask(escape_vector flags) noexcept {
- uint8x8_t narrowed = vshrn_n_u16(vreinterpretq_u16_u8(flags), 4);
- return vget_lane_u64(vreinterpret_u64_u8(narrowed), 0);
-}
-
-#endif // instruction set selection
-
-// The escape bitmask of a 16-byte block.
-simdjson_inline uint64_t escape_mask(escape_vector v) noexcept {
- return escape_bitmask(escape_flags(v));
-}
-
-// Copies n bytes with n < 16, using overlapping loads and stores. It never
-// reads more than n bytes from src, nor writes more than n bytes to dst.
-simdjson_inline void copy_lt16(char *dst, const uint8_t *src,
- size_t n) noexcept {
- if (n >= 8) {
- memcpy(dst, src, 8);
- memcpy(dst + n - 8, src + n - 8, 8);
- } else if (n >= 4) {
- memcpy(dst, src, 4);
- memcpy(dst + n - 4, src + n - 4, 4);
- } else if (n > 0) {
- dst[0] = char(src[0]);
- dst[n >> 1] = char(src[n >> 1]);
- dst[n - 1] = char(src[n - 1]);
- }
-}
-
-// Escapes the bytes of src in the range [i, blockend), given that m is the
-// (non-zero) escape mask for that range: the escape_mask_bits-wide lane of m
-// at byte k is non-zero when src[i + k] requires escaping. Returns the updated
-// output pointer.
-simdjson_never_inline char *escape_block(const uint8_t *src, char *out,
- size_t i, size_t blockend,
- uint64_t m) noexcept {
- constexpr uint64_t lane = (uint64_t(1) << escape_mask_bits) - 1;
-
- size_t pos = i; // first byte not yet copied
- while (m) {
- const size_t tz = trailing_zeroes(m);
- const size_t next = i + tz / escape_mask_bits;
- // Copy the run of safe bytes that precedes this escape.
- copy_lt16(out, src + pos, next - pos);
- out += next - pos;
- escape_json_char(char(src[next]), out);
- pos = next + 1;
- m &= ~(lane << tz);
- }
- // Copy whatever follows the last escape.
- copy_lt16(out, src + pos, blockend - pos);
- return out + (blockend - pos);
-}
-
-// Writes the escaped version of input to out, returning the number of bytes
-// written.
-simdjson_really_inline size_t write_string_escaped(const std::string_view input, char *out) {
- const size_t len = input.size();
- const uint8_t *src = reinterpret_cast<const uint8_t *>(input.data());
- const char *const initout = out;
-
- size_t i = 0;
-#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 && defined(__AVX2__)
- while (i + 32 <= len) {
- const __m256i word = _mm256_loadu_si256(reinterpret_cast<const __m256i *>(src + i));
- const __m256i flags = _mm256_or_si256(
- _mm256_or_si256(_mm256_cmpeq_epi8(word, _mm256_set1_epi8(34)), // '"'
- _mm256_cmpeq_epi8(word, _mm256_set1_epi8(92))), // '\\'
- _mm256_cmpeq_epi8(_mm256_subs_epu8(word, _mm256_set1_epi8(31)),
- _mm256_setzero_si256())); // control
- const uint32_t mask = uint32_t(_mm256_movemask_epi8(flags));
- if (simdjson_likely(mask == 0)) {
- _mm256_storeu_si256(reinterpret_cast<__m256i *>(out), word);
- out += 32;
- } else {
- for (size_t half = 0; half < 32; half += 16) {
- const uint64_t m = (mask >> half) & 0xFFFF;
- if (m == 0) {
- escape_store16(out, escape_load16(src + i + half));
- out += 16;
- } else {
- out = escape_block(src, out, i + half, i + half + 16, m);
- }
- }
- }
- i += 32;
- }
-#endif
- while (i + 16 <= len) {
- escape_vector word = escape_load16(src + i);
- escape_vector flags = escape_flags(word);
- if (simdjson_likely(!escape_any(flags))) {
- escape_store16(out, word);
- out += 16;
- } else {
- out = escape_block(src, out, i, i + 16, escape_bitmask(flags));
- }
- i += 16;
- }
- if (i < len) {
- const size_t rem = len - i;
- uint64_t m;
- if (len >= 16) {
- // The last 16 bytes of the input are in bounds. Bit k of that block's
- // mask belongs to input position len - 16 + k, so shift it down to align
- // bit 0 with position i.
- m = escape_mask(escape_load16(src + len - 16)) >>
- (escape_mask_bits * (16 - rem));
- } else if (len >= 8) {
- // Here i == 0 and rem == len. Two overlapping 8-byte loads cover
- // [0, 8) and [len - 8, len), which is the whole input since len < 16.
- uint64_t mm = escape_mask(escape_load8x2(src, src + len - 8));
- constexpr uint64_t low8 = (uint64_t(1) << (escape_mask_bits * 8)) - 1;
- m = (mm & low8) |
- ((mm >> (escape_mask_bits * 8)) << (escape_mask_bits * (len - 8)));
- } else if (len >= 4) {
- // Same idea with two overlapping 4-byte loads.
- uint64_t mm = escape_mask(escape_load4x2(src, src + len - 4));
- constexpr uint64_t low4 = (uint64_t(1) << (escape_mask_bits * 4)) - 1;
- m = (mm & low4) | (((mm >> (escape_mask_bits * 4)) & low4)
- << (escape_mask_bits * (len - 4)));
- } else {
- // Fewer than 4 bytes: at most three table lookups, no need for SIMD.
- for (size_t k = 0; k < len; k++) {
- uint8_t c = src[k];
- if (json_quotable_character[c]) {
- escape_json_char(char(c), out);
- } else {
- *out++ = char(c);
- }
- }
- return size_t(out - initout);
- }
- if (m == 0) {
- copy_lt16(out, src + i, rem);
- out += rem;
- } else {
- out = escape_block(src, out, i, len, m);
- }
- }
- return size_t(out - initout);
-}
-
-#else // SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-
-// Writes the escaped version of input to out, returning the number of bytes
-// written. Uses SIMD position finding to locate quotable characters efficiently.
-inline size_t write_string_escaped(const std::string_view input, char *out) {
- size_t mysize = input.size();
-
- // Use SIMD position finder directly - it returns mysize if no escape needed
- size_t location = find_next_json_quotable_character(input, 0);
- if (location == mysize) {
- // Fast path: no escaping needed
- memcpy(out, input.data(), input.size());
- return input.size();
- }
-
- const char *const initout = out;
- memcpy(out, input.data(), location);
- out += location;
- escape_json_char(input[location], out);
- location += 1;
- while (location < mysize) {
- size_t newlocation = find_next_json_quotable_character(input, location);
- memcpy(out, input.data() + location, newlocation - location);
- out += newlocation - location;
- location = newlocation;
- if (location == mysize) {
- break;
- }
- escape_json_char(input[location], out);
- location += 1;
- }
- return size_t(out - initout);
-}
-
-#endif // SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-#undef SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-
namespace internal {
// Sets the size of s to n without initializing the new bytes when the
@@ -58064,6 +56392,7 @@ backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) {
/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */
/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_H */
/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */
+/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */
/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */
namespace simdjson {
@@ -58100,6 +56429,9 @@ struct has_custom_serialization : std::false_type {};
namespace ppc64 {
namespace builder {
+
+using simdjson::internal::escape_best::write_string_escaped;
+
/**
* A builder for JSON strings representing documents. This is a low-level
* builder that is not meant to be used directly by end-users. Though it
@@ -58446,6 +56778,7 @@ simdjson_warn_unused error_code to_json(const Z &z, std::string &s, size_t initi
/* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */
/* amalgamation skipped (editor-only): #include "simdjson/concepts.h" */
/* amalgamation skipped (editor-only): #include "simdjson/annotations.h" */
+/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */
/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */
#if SIMDJSON_STATIC_REFLECTION
@@ -58471,7 +56804,6 @@ namespace internal {
using simdjson::internal::write_uint_jeaiii;
simdjson_inline char *write_double(char *p, double v) noexcept;
} // namespace internal
-simdjson_really_inline size_t write_string_escaped(const std::string_view input, char *out);
SIMDJSON_PUSH_DISABLE_WARNINGS
SIMDJSON_DISABLE_GCC_WARNING(-Warray-bounds)
@@ -59492,619 +57824,13 @@ simdjson_warn_unused simdjson_result<std::string> extract_fractured_json(
/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_INL_H */
/* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */
/* amalgamation skipped (editor-only): #include "simdjson/internal/jeaiii.h" */
+/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */
/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */
-/*
- * Empirically, we have found that an inlined optimization is important for
- * performance. The following macros are not ideal. We should find a better
- * way to inline the code.
- */
-
-#if defined(__SSE2__) || defined(__x86_64__) || defined(__x86_64) || \
- (defined(_M_AMD64) || defined(_M_X64) || \
- (defined(_M_IX86_FP) && _M_IX86_FP == 2))
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_SSE2
-#define SIMDJSON_EXPERIMENTAL_HAS_SSE2 1
-#endif
-#endif
-
-#if defined(__aarch64__) || defined(_M_ARM64)
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_NEON
-#define SIMDJSON_EXPERIMENTAL_HAS_NEON 1
-#endif
-#endif
-#if defined(__loongarch_sx)
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_LSX
-#define SIMDJSON_EXPERIMENTAL_HAS_LSX 1
-#endif
-#endif
-#if defined(__loongarch_asx)
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_LASX
-#define SIMDJSON_EXPERIMENTAL_HAS_LASX 1
-#endif
-#endif
-#if defined(__riscv_v_intrinsic) && __riscv_v_intrinsic >= 11000 && \
- defined(__riscv_vector)
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_RVV
-#define SIMDJSON_EXPERIMENTAL_HAS_RVV 1
-#endif
-#endif
-#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) && defined(__POWER8_VECTOR__)
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64
-#define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1
-#endif
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_NEON
-#include <arm_neon.h>
-#ifdef _MSC_VER
-#include <intrin.h>
-#endif
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_SSE2
-#include <emmintrin.h>
-#if defined(__AVX2__)
-#include <immintrin.h>
-#endif
-#ifdef _MSC_VER
-#include <intrin.h>
-#endif
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_LSX
-#include <lsxintrin.h>
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_LASX
-#include <lasxintrin.h>
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_RVV
-#include <riscv_vector.h>
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_PPC64
-#include <altivec.h>
-#ifdef bool
-#undef bool
-#endif
-#ifdef vector
-#undef vector
-#endif
-#endif
-
-
namespace simdjson {
namespace ppc64 {
namespace builder {
-static SIMDJSON_CONSTEXPR_LAMBDA std::array<uint8_t, 256>
- json_quotable_character = {
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
-
-/**
-
-A possible SWAR implementation of has_json_escapable_byte. It is not used
-because it is slower than the current implementation. It is kept here for
-reference (to show that we tried it).
-
-inline bool has_json_escapable_byte(uint64_t x) {
- uint64_t is_ascii = 0x8080808080808080ULL & ~x;
- uint64_t xor2 = x ^ 0x0202020202020202ULL;
- uint64_t lt32_or_eq34 = xor2 - 0x2121212121212121ULL;
- uint64_t sub92 = x ^ 0x5C5C5C5C5C5C5C5CULL;
- uint64_t eq92 = (sub92 - 0x0101010101010101ULL);
- return ((lt32_or_eq34 | eq92) & is_ascii) != 0;
-}
-
-**/
-
-#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 || SIMDJSON_EXPERIMENTAL_HAS_NEON
-#define SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE 1
-#endif
-
-// With block escaping (SSE2/NEON), write_string_escaped scans the input itself.
-#if !SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-// Scalar fallback for finding next quotable character
-SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline size_t
-find_next_json_quotable_character_scalar(const std::string_view view,
- size_t location) noexcept {
- for (auto pos = view.begin() + location; pos != view.end(); ++pos) {
- if (json_quotable_character[static_cast<uint8_t>(*pos)]) {
- return pos - view.begin();
- }
- }
- return size_t(view.size());
-}
-
-// SIMD-accelerated position finding that directly locates the first quotable
-// character, combining detection and position extraction in a single pass to
-// minimize redundant work.
-#if SIMDJSON_EXPERIMENTAL_HAS_LASX
-simdjson_inline size_t
-find_next_json_quotable_character(const std::string_view view,
- size_t location) noexcept {
- const size_t len = view.size();
- const uint8_t *ptr =
- reinterpret_cast<const uint8_t *>(view.data()) + location;
- size_t remaining = len - location;
-
- // SIMD constants for characters requiring escape
- __m256i v34 = __lasx_xvreplgr2vr_b(34); // '"'
- __m256i v92 = __lasx_xvreplgr2vr_b(92); // '\\'
- __m256i v32 = __lasx_xvreplgr2vr_b(32); // control char threshold
-
- while (remaining >= 32) {
- __m256i word = __lasx_xvld(ptr, 0);
-
- // Check for quotable characters: '"', '\\', or control chars (< 32)
- __m256i needs_escape = __lasx_xvseq_b(word, v34);
- needs_escape = __lasx_xvor_v(needs_escape, __lasx_xvseq_b(word, v92));
- needs_escape = __lasx_xvor_v(needs_escape, __lasx_xvslt_bu(word, v32));
-
- if (!__lasx_xbz_v(needs_escape)) {
- // Found a quotable character - locate it via the four 64-bit lanes
- uint64_t lane0 = __lasx_xvpickve2gr_du(needs_escape, 0);
- uint64_t lane1 = __lasx_xvpickve2gr_du(needs_escape, 1);
- uint64_t lane2 = __lasx_xvpickve2gr_du(needs_escape, 2);
- uint64_t lane3 = __lasx_xvpickve2gr_du(needs_escape, 3);
- size_t offset = ptr - reinterpret_cast<const uint8_t *>(view.data());
- if (lane0 != 0) {
- return offset + trailing_zeroes(lane0) / 8;
- } else if (lane1 != 0) {
- return offset + 8 + trailing_zeroes(lane1) / 8;
- } else if (lane2 != 0) {
- return offset + 16 + trailing_zeroes(lane2) / 8;
- } else {
- return offset + 24 + trailing_zeroes(lane3) / 8;
- }
- }
- ptr += 32;
- remaining -= 32;
- }
- size_t current = len - remaining;
- return find_next_json_quotable_character_scalar(view, current);
-}
-#elif SIMDJSON_EXPERIMENTAL_HAS_LSX
-simdjson_inline size_t
-find_next_json_quotable_character(const std::string_view view,
- size_t location) noexcept {
- const size_t len = view.size();
- const uint8_t *ptr =
- reinterpret_cast<const uint8_t *>(view.data()) + location;
- size_t remaining = len - location;
-
- //SIMD constants for characters requiring escape
- __m128i v34 = __lsx_vreplgr2vr_b(34); // '"'
- __m128i v92 = __lsx_vreplgr2vr_b(92); // '\\'
- __m128i v32 = __lsx_vreplgr2vr_b(32); // control char threshold
-
- while (remaining >= 16){
- __m128i word = __lsx_vld(ptr, 0);
-
- //Check for the quotable characters: '"', '\\', or control char (<32)
- __m128i needs_escape = __lsx_vseq_b(word, v34);
- needs_escape = __lsx_vor_v(needs_escape, __lsx_vseq_b(word, v92));
- needs_escape = __lsx_vor_v(needs_escape, __lsx_vslt_bu(word, v32));
-
- if (!__lsx_bz_v(needs_escape)){
-
- //Found quotable character - extract exact byte position
- uint64_t lo = __lsx_vpickve2gr_du(needs_escape,0);
- uint64_t hi = __lsx_vpickve2gr_du(needs_escape,1);
- size_t offset = ptr - reinterpret_cast<const uint8_t *>(view.data());
- if ( lo != 0) {
- return offset + trailing_zeroes(lo) / 8;
- } else {
- return offset + 8 + trailing_zeroes(hi) / 8;
- }
- }
- ptr += 16;
- remaining -= 16;
- }
- size_t current = len - remaining;
- return find_next_json_quotable_character_scalar(view, current);
-}
-#elif SIMDJSON_EXPERIMENTAL_HAS_RVV
-simdjson_inline size_t
-find_next_json_quotable_character(const std::string_view view,
- size_t location) noexcept {
- const size_t len = view.size();
- const uint8_t *ptr =
- reinterpret_cast<const uint8_t *>(view.data()) + location;
- size_t remaining = len - location;
-
- while (remaining > 0) {
- size_t vl = __riscv_vsetvl_e8m1(remaining);
- vuint8m1_t word = __riscv_vle8_v_u8m1(ptr, vl);
-
- // Check for quotable characters: '"', '\\', or control chars (< 32)
- vbool8_t needs_escape = __riscv_vmseq(word, (uint8_t)34, vl);
- needs_escape = __riscv_vmor(needs_escape,
- __riscv_vmseq(word, (uint8_t)92, vl), vl);
- needs_escape = __riscv_vmor(needs_escape,
- __riscv_vmsltu(word, (uint8_t)32, vl), vl);
-
- long first = __riscv_vfirst(needs_escape, vl);
- if (first >= 0) {
- size_t offset = ptr - reinterpret_cast<const uint8_t *>(view.data());
- return offset + first;
- }
- ptr += vl;
- remaining -= vl;
- }
-
- return len;
-}
-#elif SIMDJSON_EXPERIMENTAL_HAS_PPC64
-simdjson_inline size_t
-find_next_json_quotable_character(const std::string_view view,
- size_t location) noexcept {
- const size_t len = view.size();
- const uint8_t *ptr =
- reinterpret_cast<const uint8_t *>(view.data()) + location;
- size_t remaining = len - location;
-
- // SIMD constants for characters requiring escape
- __vector unsigned char v34 = vec_splats((unsigned char)34); // '"'
- __vector unsigned char v92 = vec_splats((unsigned char)92); // '\\'
- __vector unsigned char v32 = vec_splats((unsigned char)32); // control char threshold
-
- // Bitmask for vec_vbpermq to extract one bit per byte
- const __vector unsigned char perm_mask = {0x78, 0x70, 0x68, 0x60, 0x58, 0x50,
- 0x48, 0x40, 0x38, 0x30, 0x28, 0x20,
- 0x18, 0x10, 0x08, 0x00};
-
- while (remaining >= 16) {
- __vector unsigned char word =
- vec_vsx_ld(0, reinterpret_cast<const unsigned char *>(ptr));
-
- // Check for quotable characters: '"', '\\', or control chars (< 32)
- __vector unsigned char needs_escape =
- (__vector unsigned char)vec_cmpeq(word, v34);
- needs_escape = vec_or(needs_escape,
- (__vector unsigned char)vec_cmpeq(word, v92));
- needs_escape = vec_or(needs_escape,
- (__vector unsigned char)vec_cmplt(word, v32));
-
- __vector unsigned long long result =
- (__vector unsigned long long)vec_vbpermq(needs_escape, perm_mask);
-#ifdef __LITTLE_ENDIAN__
- unsigned int mask = static_cast<unsigned int>(result[1]);
-#else
- unsigned int mask = static_cast<unsigned int>(result[0]);
-#endif
- if (mask != 0) {
- size_t offset = ptr - reinterpret_cast<const uint8_t *>(view.data());
- return offset + __builtin_ctz(mask);
- }
- ptr += 16;
- remaining -= 16;
- }
-
- // Scalar fallback for remaining bytes
- size_t current = len - remaining;
- return find_next_json_quotable_character_scalar(view, current);
-}
-#else
-SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline size_t
-find_next_json_quotable_character(const std::string_view view,
- size_t location) noexcept {
- return find_next_json_quotable_character_scalar(view, location);
-}
-#endif
-#endif // !SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-
-SIMDJSON_CONSTEXPR_LAMBDA static std::string_view control_chars[] = {
- "\\u0000", "\\u0001", "\\u0002", "\\u0003", "\\u0004", "\\u0005", "\\u0006",
- "\\u0007", "\\b", "\\t", "\\n", "\\u000b", "\\f", "\\r",
- "\\u000e", "\\u000f", "\\u0010", "\\u0011", "\\u0012", "\\u0013", "\\u0014",
- "\\u0015", "\\u0016", "\\u0017", "\\u0018", "\\u0019", "\\u001a", "\\u001b",
- "\\u001c", "\\u001d", "\\u001e", "\\u001f"};
-
-// All Unicode characters may be placed within the quotation marks, except for
-// the characters that MUST be escaped: quotation mark, reverse solidus, and the
-// control characters (U+0000 through U+001F). There are two-character sequence
-// escape representations of some popular characters:
-// \", \\, \b, \f, \n, \r, \t.
-SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline void escape_json_char(char c, char *&out) {
- if (c == '"') {
- memcpy(out, "\\\"", 2);
- out += 2;
- } else if (c == '\\') {
- memcpy(out, "\\\\", 2);
- out += 2;
- } else {
- std::string_view v = control_chars[uint8_t(c)];
- memcpy(out, v.data(), v.size());
- out += v.size();
- }
-}
-
-#if SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-
-#if SIMDJSON_EXPERIMENTAL_HAS_SSE2
-
-using escape_vector = __m128i;
-// Mask bits that each input byte contributes to escape_bitmask().
-static constexpr unsigned escape_mask_bits = 1;
-
-simdjson_inline escape_vector escape_load16(const uint8_t *p) noexcept {
- return _mm_loadu_si128(reinterpret_cast<const __m128i *>(p));
-}
-
-simdjson_inline void escape_store16(char *out, escape_vector v) noexcept {
- _mm_storeu_si128(reinterpret_cast<__m128i *>(out), v);
-}
-
-// Builds a vector whose bytes 0..7 come from a and bytes 8..15 from b.
-simdjson_inline escape_vector escape_load8x2(const uint8_t *a,
- const uint8_t *b) noexcept {
- return _mm_unpacklo_epi64(
- _mm_loadl_epi64(reinterpret_cast<const __m128i *>(a)),
- _mm_loadl_epi64(reinterpret_cast<const __m128i *>(b)));
-}
-
-// Builds a vector whose bytes 0..3 come from a and bytes 4..7 from b. The
-// upper half is unspecified; callers only look at the low eight bits.
-simdjson_inline escape_vector escape_load4x2(const uint8_t *a,
- const uint8_t *b) noexcept {
- int32_t a32, b32;
- memcpy(&a32, a, 4);
- memcpy(&b32, b, 4);
- return _mm_unpacklo_epi32(_mm_cvtsi32_si128(a32), _mm_cvtsi32_si128(b32));
-}
-
-// Sets every bit of byte k when byte k of v is a quotable character ('"',
-// '\\' or a control character).
-simdjson_inline escape_vector escape_flags(escape_vector v) noexcept {
- const __m128i v34 = _mm_set1_epi8(34); // '"'
- const __m128i v92 = _mm_set1_epi8(92); // '\\'
- const __m128i v31 = _mm_set1_epi8(31); // for control char detection
- __m128i needs_escape = _mm_cmpeq_epi8(v, v34);
- needs_escape = _mm_or_si128(needs_escape, _mm_cmpeq_epi8(v, v92));
- return _mm_or_si128(
- needs_escape, _mm_cmpeq_epi8(_mm_subs_epu8(v, v31), _mm_setzero_si128()));
-}
-
-// True when any byte needs escaping. Kept separate from escape_bitmask
-// because some instruction sets can answer it without leaving the vector
-// register file; on SSE2 the compiler folds the two together.
-simdjson_inline bool escape_any(escape_vector flags) noexcept {
- return _mm_movemask_epi8(flags) != 0;
-}
-
-// A 16-bit mask with bit k set when byte k needed escaping.
-simdjson_inline uint64_t escape_bitmask(escape_vector flags) noexcept {
- return uint64_t(uint32_t(_mm_movemask_epi8(flags)));
-}
-
-#else // SIMDJSON_EXPERIMENTAL_HAS_NEON
-
-using escape_vector = uint8x16_t;
-static constexpr unsigned escape_mask_bits = 4;
-
-simdjson_inline escape_vector escape_load16(const uint8_t *p) noexcept {
- return vld1q_u8(p);
-}
-
-simdjson_inline void escape_store16(char *out, escape_vector v) noexcept {
- vst1q_u8(reinterpret_cast<uint8_t *>(out), v);
-}
-
-simdjson_inline escape_vector escape_load8x2(const uint8_t *a,
- const uint8_t *b) noexcept {
- uint64_t a64, b64;
- memcpy(&a64, a, 8);
- memcpy(&b64, b, 8);
- return vreinterpretq_u8_u64(vsetq_lane_u64(b64, vdupq_n_u64(a64), 1));
-}
-
-simdjson_inline escape_vector escape_load4x2(const uint8_t *a,
- const uint8_t *b) noexcept {
- uint32_t a32, b32;
- memcpy(&a32, a, 4);
- memcpy(&b32, b, 4);
- return vreinterpretq_u8_u32(vsetq_lane_u32(b32, vdupq_n_u32(a32), 1));
-}
-
-simdjson_inline escape_vector escape_flags(escape_vector v) noexcept {
- uint8x16_t needs_escape = vceqq_u8(v, vdupq_n_u8(34)); // '"'
- needs_escape = vorrq_u8(needs_escape, vceqq_u8(v, vdupq_n_u8(92))); // '\\'
- return vorrq_u8(needs_escape, vcltq_u8(v, vdupq_n_u8(32)));
-}
-
-simdjson_inline bool escape_any(escape_vector flags) noexcept {
- return vmaxvq_u32(vreinterpretq_u32_u8(flags)) != 0;
-}
-
-// Four bits per byte rather than one: escape_block and the tail paths scale
-// their shifts by escape_mask_bits to match.
-simdjson_inline uint64_t escape_bitmask(escape_vector flags) noexcept {
- uint8x8_t narrowed = vshrn_n_u16(vreinterpretq_u16_u8(flags), 4);
- return vget_lane_u64(vreinterpret_u64_u8(narrowed), 0);
-}
-
-#endif // instruction set selection
-
-// The escape bitmask of a 16-byte block.
-simdjson_inline uint64_t escape_mask(escape_vector v) noexcept {
- return escape_bitmask(escape_flags(v));
-}
-
-// Copies n bytes with n < 16, using overlapping loads and stores. It never
-// reads more than n bytes from src, nor writes more than n bytes to dst.
-simdjson_inline void copy_lt16(char *dst, const uint8_t *src,
- size_t n) noexcept {
- if (n >= 8) {
- memcpy(dst, src, 8);
- memcpy(dst + n - 8, src + n - 8, 8);
- } else if (n >= 4) {
- memcpy(dst, src, 4);
- memcpy(dst + n - 4, src + n - 4, 4);
- } else if (n > 0) {
- dst[0] = char(src[0]);
- dst[n >> 1] = char(src[n >> 1]);
- dst[n - 1] = char(src[n - 1]);
- }
-}
-
-// Escapes the bytes of src in the range [i, blockend), given that m is the
-// (non-zero) escape mask for that range: the escape_mask_bits-wide lane of m
-// at byte k is non-zero when src[i + k] requires escaping. Returns the updated
-// output pointer.
-simdjson_never_inline char *escape_block(const uint8_t *src, char *out,
- size_t i, size_t blockend,
- uint64_t m) noexcept {
- constexpr uint64_t lane = (uint64_t(1) << escape_mask_bits) - 1;
-
- size_t pos = i; // first byte not yet copied
- while (m) {
- const size_t tz = trailing_zeroes(m);
- const size_t next = i + tz / escape_mask_bits;
- // Copy the run of safe bytes that precedes this escape.
- copy_lt16(out, src + pos, next - pos);
- out += next - pos;
- escape_json_char(char(src[next]), out);
- pos = next + 1;
- m &= ~(lane << tz);
- }
- // Copy whatever follows the last escape.
- copy_lt16(out, src + pos, blockend - pos);
- return out + (blockend - pos);
-}
-
-// Writes the escaped version of input to out, returning the number of bytes
-// written.
-simdjson_really_inline size_t write_string_escaped(const std::string_view input, char *out) {
- const size_t len = input.size();
- const uint8_t *src = reinterpret_cast<const uint8_t *>(input.data());
- const char *const initout = out;
-
- size_t i = 0;
-#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 && defined(__AVX2__)
- while (i + 32 <= len) {
- const __m256i word = _mm256_loadu_si256(reinterpret_cast<const __m256i *>(src + i));
- const __m256i flags = _mm256_or_si256(
- _mm256_or_si256(_mm256_cmpeq_epi8(word, _mm256_set1_epi8(34)), // '"'
- _mm256_cmpeq_epi8(word, _mm256_set1_epi8(92))), // '\\'
- _mm256_cmpeq_epi8(_mm256_subs_epu8(word, _mm256_set1_epi8(31)),
- _mm256_setzero_si256())); // control
- const uint32_t mask = uint32_t(_mm256_movemask_epi8(flags));
- if (simdjson_likely(mask == 0)) {
- _mm256_storeu_si256(reinterpret_cast<__m256i *>(out), word);
- out += 32;
- } else {
- for (size_t half = 0; half < 32; half += 16) {
- const uint64_t m = (mask >> half) & 0xFFFF;
- if (m == 0) {
- escape_store16(out, escape_load16(src + i + half));
- out += 16;
- } else {
- out = escape_block(src, out, i + half, i + half + 16, m);
- }
- }
- }
- i += 32;
- }
-#endif
- while (i + 16 <= len) {
- escape_vector word = escape_load16(src + i);
- escape_vector flags = escape_flags(word);
- if (simdjson_likely(!escape_any(flags))) {
- escape_store16(out, word);
- out += 16;
- } else {
- out = escape_block(src, out, i, i + 16, escape_bitmask(flags));
- }
- i += 16;
- }
- if (i < len) {
- const size_t rem = len - i;
- uint64_t m;
- if (len >= 16) {
- // The last 16 bytes of the input are in bounds. Bit k of that block's
- // mask belongs to input position len - 16 + k, so shift it down to align
- // bit 0 with position i.
- m = escape_mask(escape_load16(src + len - 16)) >>
- (escape_mask_bits * (16 - rem));
- } else if (len >= 8) {
- // Here i == 0 and rem == len. Two overlapping 8-byte loads cover
- // [0, 8) and [len - 8, len), which is the whole input since len < 16.
- uint64_t mm = escape_mask(escape_load8x2(src, src + len - 8));
- constexpr uint64_t low8 = (uint64_t(1) << (escape_mask_bits * 8)) - 1;
- m = (mm & low8) |
- ((mm >> (escape_mask_bits * 8)) << (escape_mask_bits * (len - 8)));
- } else if (len >= 4) {
- // Same idea with two overlapping 4-byte loads.
- uint64_t mm = escape_mask(escape_load4x2(src, src + len - 4));
- constexpr uint64_t low4 = (uint64_t(1) << (escape_mask_bits * 4)) - 1;
- m = (mm & low4) | (((mm >> (escape_mask_bits * 4)) & low4)
- << (escape_mask_bits * (len - 4)));
- } else {
- // Fewer than 4 bytes: at most three table lookups, no need for SIMD.
- for (size_t k = 0; k < len; k++) {
- uint8_t c = src[k];
- if (json_quotable_character[c]) {
- escape_json_char(char(c), out);
- } else {
- *out++ = char(c);
- }
- }
- return size_t(out - initout);
- }
- if (m == 0) {
- copy_lt16(out, src + i, rem);
- out += rem;
- } else {
- out = escape_block(src, out, i, len, m);
- }
- }
- return size_t(out - initout);
-}
-
-#else // SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-
-// Writes the escaped version of input to out, returning the number of bytes
-// written. Uses SIMD position finding to locate quotable characters efficiently.
-inline size_t write_string_escaped(const std::string_view input, char *out) {
- size_t mysize = input.size();
-
- // Use SIMD position finder directly - it returns mysize if no escape needed
- size_t location = find_next_json_quotable_character(input, 0);
- if (location == mysize) {
- // Fast path: no escaping needed
- memcpy(out, input.data(), input.size());
- return input.size();
- }
-
- const char *const initout = out;
- memcpy(out, input.data(), location);
- out += location;
- escape_json_char(input[location], out);
- location += 1;
- while (location < mysize) {
- size_t newlocation = find_next_json_quotable_character(input, location);
- memcpy(out, input.data() + location, newlocation - location);
- out += newlocation - location;
- location = newlocation;
- if (location == mysize) {
- break;
- }
- escape_json_char(input[location], out);
- location += 1;
- }
- return size_t(out - initout);
-}
-
-#endif // SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-#undef SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-
namespace internal {
// Sets the size of s to n without initializing the new bytes when the
@@ -61851,6 +59577,7 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin
/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */
/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_H */
/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */
+/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */
/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */
namespace simdjson {
@@ -61887,6 +59614,9 @@ struct has_custom_serialization : std::false_type {};
namespace westmere {
namespace builder {
+
+using simdjson::internal::escape_best::write_string_escaped;
+
/**
* A builder for JSON strings representing documents. This is a low-level
* builder that is not meant to be used directly by end-users. Though it
@@ -62233,6 +59963,7 @@ simdjson_warn_unused error_code to_json(const Z &z, std::string &s, size_t initi
/* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */
/* amalgamation skipped (editor-only): #include "simdjson/concepts.h" */
/* amalgamation skipped (editor-only): #include "simdjson/annotations.h" */
+/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */
/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */
#if SIMDJSON_STATIC_REFLECTION
@@ -62258,7 +59989,6 @@ namespace internal {
using simdjson::internal::write_uint_jeaiii;
simdjson_inline char *write_double(char *p, double v) noexcept;
} // namespace internal
-simdjson_really_inline size_t write_string_escaped(const std::string_view input, char *out);
SIMDJSON_PUSH_DISABLE_WARNINGS
SIMDJSON_DISABLE_GCC_WARNING(-Warray-bounds)
@@ -63279,619 +61009,13 @@ simdjson_warn_unused simdjson_result<std::string> extract_fractured_json(
/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_INL_H */
/* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */
/* amalgamation skipped (editor-only): #include "simdjson/internal/jeaiii.h" */
+/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */
/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */
-/*
- * Empirically, we have found that an inlined optimization is important for
- * performance. The following macros are not ideal. We should find a better
- * way to inline the code.
- */
-
-#if defined(__SSE2__) || defined(__x86_64__) || defined(__x86_64) || \
- (defined(_M_AMD64) || defined(_M_X64) || \
- (defined(_M_IX86_FP) && _M_IX86_FP == 2))
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_SSE2
-#define SIMDJSON_EXPERIMENTAL_HAS_SSE2 1
-#endif
-#endif
-
-#if defined(__aarch64__) || defined(_M_ARM64)
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_NEON
-#define SIMDJSON_EXPERIMENTAL_HAS_NEON 1
-#endif
-#endif
-#if defined(__loongarch_sx)
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_LSX
-#define SIMDJSON_EXPERIMENTAL_HAS_LSX 1
-#endif
-#endif
-#if defined(__loongarch_asx)
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_LASX
-#define SIMDJSON_EXPERIMENTAL_HAS_LASX 1
-#endif
-#endif
-#if defined(__riscv_v_intrinsic) && __riscv_v_intrinsic >= 11000 && \
- defined(__riscv_vector)
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_RVV
-#define SIMDJSON_EXPERIMENTAL_HAS_RVV 1
-#endif
-#endif
-#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) && defined(__POWER8_VECTOR__)
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64
-#define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1
-#endif
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_NEON
-#include <arm_neon.h>
-#ifdef _MSC_VER
-#include <intrin.h>
-#endif
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_SSE2
-#include <emmintrin.h>
-#if defined(__AVX2__)
-#include <immintrin.h>
-#endif
-#ifdef _MSC_VER
-#include <intrin.h>
-#endif
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_LSX
-#include <lsxintrin.h>
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_LASX
-#include <lasxintrin.h>
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_RVV
-#include <riscv_vector.h>
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_PPC64
-#include <altivec.h>
-#ifdef bool
-#undef bool
-#endif
-#ifdef vector
-#undef vector
-#endif
-#endif
-
-
namespace simdjson {
namespace westmere {
namespace builder {
-static SIMDJSON_CONSTEXPR_LAMBDA std::array<uint8_t, 256>
- json_quotable_character = {
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
-
-/**
-
-A possible SWAR implementation of has_json_escapable_byte. It is not used
-because it is slower than the current implementation. It is kept here for
-reference (to show that we tried it).
-
-inline bool has_json_escapable_byte(uint64_t x) {
- uint64_t is_ascii = 0x8080808080808080ULL & ~x;
- uint64_t xor2 = x ^ 0x0202020202020202ULL;
- uint64_t lt32_or_eq34 = xor2 - 0x2121212121212121ULL;
- uint64_t sub92 = x ^ 0x5C5C5C5C5C5C5C5CULL;
- uint64_t eq92 = (sub92 - 0x0101010101010101ULL);
- return ((lt32_or_eq34 | eq92) & is_ascii) != 0;
-}
-
-**/
-
-#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 || SIMDJSON_EXPERIMENTAL_HAS_NEON
-#define SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE 1
-#endif
-
-// With block escaping (SSE2/NEON), write_string_escaped scans the input itself.
-#if !SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-// Scalar fallback for finding next quotable character
-SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline size_t
-find_next_json_quotable_character_scalar(const std::string_view view,
- size_t location) noexcept {
- for (auto pos = view.begin() + location; pos != view.end(); ++pos) {
- if (json_quotable_character[static_cast<uint8_t>(*pos)]) {
- return pos - view.begin();
- }
- }
- return size_t(view.size());
-}
-
-// SIMD-accelerated position finding that directly locates the first quotable
-// character, combining detection and position extraction in a single pass to
-// minimize redundant work.
-#if SIMDJSON_EXPERIMENTAL_HAS_LASX
-simdjson_inline size_t
-find_next_json_quotable_character(const std::string_view view,
- size_t location) noexcept {
- const size_t len = view.size();
- const uint8_t *ptr =
- reinterpret_cast<const uint8_t *>(view.data()) + location;
- size_t remaining = len - location;
-
- // SIMD constants for characters requiring escape
- __m256i v34 = __lasx_xvreplgr2vr_b(34); // '"'
- __m256i v92 = __lasx_xvreplgr2vr_b(92); // '\\'
- __m256i v32 = __lasx_xvreplgr2vr_b(32); // control char threshold
-
- while (remaining >= 32) {
- __m256i word = __lasx_xvld(ptr, 0);
-
- // Check for quotable characters: '"', '\\', or control chars (< 32)
- __m256i needs_escape = __lasx_xvseq_b(word, v34);
- needs_escape = __lasx_xvor_v(needs_escape, __lasx_xvseq_b(word, v92));
- needs_escape = __lasx_xvor_v(needs_escape, __lasx_xvslt_bu(word, v32));
-
- if (!__lasx_xbz_v(needs_escape)) {
- // Found a quotable character - locate it via the four 64-bit lanes
- uint64_t lane0 = __lasx_xvpickve2gr_du(needs_escape, 0);
- uint64_t lane1 = __lasx_xvpickve2gr_du(needs_escape, 1);
- uint64_t lane2 = __lasx_xvpickve2gr_du(needs_escape, 2);
- uint64_t lane3 = __lasx_xvpickve2gr_du(needs_escape, 3);
- size_t offset = ptr - reinterpret_cast<const uint8_t *>(view.data());
- if (lane0 != 0) {
- return offset + trailing_zeroes(lane0) / 8;
- } else if (lane1 != 0) {
- return offset + 8 + trailing_zeroes(lane1) / 8;
- } else if (lane2 != 0) {
- return offset + 16 + trailing_zeroes(lane2) / 8;
- } else {
- return offset + 24 + trailing_zeroes(lane3) / 8;
- }
- }
- ptr += 32;
- remaining -= 32;
- }
- size_t current = len - remaining;
- return find_next_json_quotable_character_scalar(view, current);
-}
-#elif SIMDJSON_EXPERIMENTAL_HAS_LSX
-simdjson_inline size_t
-find_next_json_quotable_character(const std::string_view view,
- size_t location) noexcept {
- const size_t len = view.size();
- const uint8_t *ptr =
- reinterpret_cast<const uint8_t *>(view.data()) + location;
- size_t remaining = len - location;
-
- //SIMD constants for characters requiring escape
- __m128i v34 = __lsx_vreplgr2vr_b(34); // '"'
- __m128i v92 = __lsx_vreplgr2vr_b(92); // '\\'
- __m128i v32 = __lsx_vreplgr2vr_b(32); // control char threshold
-
- while (remaining >= 16){
- __m128i word = __lsx_vld(ptr, 0);
-
- //Check for the quotable characters: '"', '\\', or control char (<32)
- __m128i needs_escape = __lsx_vseq_b(word, v34);
- needs_escape = __lsx_vor_v(needs_escape, __lsx_vseq_b(word, v92));
- needs_escape = __lsx_vor_v(needs_escape, __lsx_vslt_bu(word, v32));
-
- if (!__lsx_bz_v(needs_escape)){
-
- //Found quotable character - extract exact byte position
- uint64_t lo = __lsx_vpickve2gr_du(needs_escape,0);
- uint64_t hi = __lsx_vpickve2gr_du(needs_escape,1);
- size_t offset = ptr - reinterpret_cast<const uint8_t *>(view.data());
- if ( lo != 0) {
- return offset + trailing_zeroes(lo) / 8;
- } else {
- return offset + 8 + trailing_zeroes(hi) / 8;
- }
- }
- ptr += 16;
- remaining -= 16;
- }
- size_t current = len - remaining;
- return find_next_json_quotable_character_scalar(view, current);
-}
-#elif SIMDJSON_EXPERIMENTAL_HAS_RVV
-simdjson_inline size_t
-find_next_json_quotable_character(const std::string_view view,
- size_t location) noexcept {
- const size_t len = view.size();
- const uint8_t *ptr =
- reinterpret_cast<const uint8_t *>(view.data()) + location;
- size_t remaining = len - location;
-
- while (remaining > 0) {
- size_t vl = __riscv_vsetvl_e8m1(remaining);
- vuint8m1_t word = __riscv_vle8_v_u8m1(ptr, vl);
-
- // Check for quotable characters: '"', '\\', or control chars (< 32)
- vbool8_t needs_escape = __riscv_vmseq(word, (uint8_t)34, vl);
- needs_escape = __riscv_vmor(needs_escape,
- __riscv_vmseq(word, (uint8_t)92, vl), vl);
- needs_escape = __riscv_vmor(needs_escape,
- __riscv_vmsltu(word, (uint8_t)32, vl), vl);
-
- long first = __riscv_vfirst(needs_escape, vl);
- if (first >= 0) {
- size_t offset = ptr - reinterpret_cast<const uint8_t *>(view.data());
- return offset + first;
- }
- ptr += vl;
- remaining -= vl;
- }
-
- return len;
-}
-#elif SIMDJSON_EXPERIMENTAL_HAS_PPC64
-simdjson_inline size_t
-find_next_json_quotable_character(const std::string_view view,
- size_t location) noexcept {
- const size_t len = view.size();
- const uint8_t *ptr =
- reinterpret_cast<const uint8_t *>(view.data()) + location;
- size_t remaining = len - location;
-
- // SIMD constants for characters requiring escape
- __vector unsigned char v34 = vec_splats((unsigned char)34); // '"'
- __vector unsigned char v92 = vec_splats((unsigned char)92); // '\\'
- __vector unsigned char v32 = vec_splats((unsigned char)32); // control char threshold
-
- // Bitmask for vec_vbpermq to extract one bit per byte
- const __vector unsigned char perm_mask = {0x78, 0x70, 0x68, 0x60, 0x58, 0x50,
- 0x48, 0x40, 0x38, 0x30, 0x28, 0x20,
- 0x18, 0x10, 0x08, 0x00};
-
- while (remaining >= 16) {
- __vector unsigned char word =
- vec_vsx_ld(0, reinterpret_cast<const unsigned char *>(ptr));
-
- // Check for quotable characters: '"', '\\', or control chars (< 32)
- __vector unsigned char needs_escape =
- (__vector unsigned char)vec_cmpeq(word, v34);
- needs_escape = vec_or(needs_escape,
- (__vector unsigned char)vec_cmpeq(word, v92));
- needs_escape = vec_or(needs_escape,
- (__vector unsigned char)vec_cmplt(word, v32));
-
- __vector unsigned long long result =
- (__vector unsigned long long)vec_vbpermq(needs_escape, perm_mask);
-#ifdef __LITTLE_ENDIAN__
- unsigned int mask = static_cast<unsigned int>(result[1]);
-#else
- unsigned int mask = static_cast<unsigned int>(result[0]);
-#endif
- if (mask != 0) {
- size_t offset = ptr - reinterpret_cast<const uint8_t *>(view.data());
- return offset + __builtin_ctz(mask);
- }
- ptr += 16;
- remaining -= 16;
- }
-
- // Scalar fallback for remaining bytes
- size_t current = len - remaining;
- return find_next_json_quotable_character_scalar(view, current);
-}
-#else
-SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline size_t
-find_next_json_quotable_character(const std::string_view view,
- size_t location) noexcept {
- return find_next_json_quotable_character_scalar(view, location);
-}
-#endif
-#endif // !SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-
-SIMDJSON_CONSTEXPR_LAMBDA static std::string_view control_chars[] = {
- "\\u0000", "\\u0001", "\\u0002", "\\u0003", "\\u0004", "\\u0005", "\\u0006",
- "\\u0007", "\\b", "\\t", "\\n", "\\u000b", "\\f", "\\r",
- "\\u000e", "\\u000f", "\\u0010", "\\u0011", "\\u0012", "\\u0013", "\\u0014",
- "\\u0015", "\\u0016", "\\u0017", "\\u0018", "\\u0019", "\\u001a", "\\u001b",
- "\\u001c", "\\u001d", "\\u001e", "\\u001f"};
-
-// All Unicode characters may be placed within the quotation marks, except for
-// the characters that MUST be escaped: quotation mark, reverse solidus, and the
-// control characters (U+0000 through U+001F). There are two-character sequence
-// escape representations of some popular characters:
-// \", \\, \b, \f, \n, \r, \t.
-SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline void escape_json_char(char c, char *&out) {
- if (c == '"') {
- memcpy(out, "\\\"", 2);
- out += 2;
- } else if (c == '\\') {
- memcpy(out, "\\\\", 2);
- out += 2;
- } else {
- std::string_view v = control_chars[uint8_t(c)];
- memcpy(out, v.data(), v.size());
- out += v.size();
- }
-}
-
-#if SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-
-#if SIMDJSON_EXPERIMENTAL_HAS_SSE2
-
-using escape_vector = __m128i;
-// Mask bits that each input byte contributes to escape_bitmask().
-static constexpr unsigned escape_mask_bits = 1;
-
-simdjson_inline escape_vector escape_load16(const uint8_t *p) noexcept {
- return _mm_loadu_si128(reinterpret_cast<const __m128i *>(p));
-}
-
-simdjson_inline void escape_store16(char *out, escape_vector v) noexcept {
- _mm_storeu_si128(reinterpret_cast<__m128i *>(out), v);
-}
-
-// Builds a vector whose bytes 0..7 come from a and bytes 8..15 from b.
-simdjson_inline escape_vector escape_load8x2(const uint8_t *a,
- const uint8_t *b) noexcept {
- return _mm_unpacklo_epi64(
- _mm_loadl_epi64(reinterpret_cast<const __m128i *>(a)),
- _mm_loadl_epi64(reinterpret_cast<const __m128i *>(b)));
-}
-
-// Builds a vector whose bytes 0..3 come from a and bytes 4..7 from b. The
-// upper half is unspecified; callers only look at the low eight bits.
-simdjson_inline escape_vector escape_load4x2(const uint8_t *a,
- const uint8_t *b) noexcept {
- int32_t a32, b32;
- memcpy(&a32, a, 4);
- memcpy(&b32, b, 4);
- return _mm_unpacklo_epi32(_mm_cvtsi32_si128(a32), _mm_cvtsi32_si128(b32));
-}
-
-// Sets every bit of byte k when byte k of v is a quotable character ('"',
-// '\\' or a control character).
-simdjson_inline escape_vector escape_flags(escape_vector v) noexcept {
- const __m128i v34 = _mm_set1_epi8(34); // '"'
- const __m128i v92 = _mm_set1_epi8(92); // '\\'
- const __m128i v31 = _mm_set1_epi8(31); // for control char detection
- __m128i needs_escape = _mm_cmpeq_epi8(v, v34);
- needs_escape = _mm_or_si128(needs_escape, _mm_cmpeq_epi8(v, v92));
- return _mm_or_si128(
- needs_escape, _mm_cmpeq_epi8(_mm_subs_epu8(v, v31), _mm_setzero_si128()));
-}
-
-// True when any byte needs escaping. Kept separate from escape_bitmask
-// because some instruction sets can answer it without leaving the vector
-// register file; on SSE2 the compiler folds the two together.
-simdjson_inline bool escape_any(escape_vector flags) noexcept {
- return _mm_movemask_epi8(flags) != 0;
-}
-
-// A 16-bit mask with bit k set when byte k needed escaping.
-simdjson_inline uint64_t escape_bitmask(escape_vector flags) noexcept {
- return uint64_t(uint32_t(_mm_movemask_epi8(flags)));
-}
-
-#else // SIMDJSON_EXPERIMENTAL_HAS_NEON
-
-using escape_vector = uint8x16_t;
-static constexpr unsigned escape_mask_bits = 4;
-
-simdjson_inline escape_vector escape_load16(const uint8_t *p) noexcept {
- return vld1q_u8(p);
-}
-
-simdjson_inline void escape_store16(char *out, escape_vector v) noexcept {
- vst1q_u8(reinterpret_cast<uint8_t *>(out), v);
-}
-
-simdjson_inline escape_vector escape_load8x2(const uint8_t *a,
- const uint8_t *b) noexcept {
- uint64_t a64, b64;
- memcpy(&a64, a, 8);
- memcpy(&b64, b, 8);
- return vreinterpretq_u8_u64(vsetq_lane_u64(b64, vdupq_n_u64(a64), 1));
-}
-
-simdjson_inline escape_vector escape_load4x2(const uint8_t *a,
- const uint8_t *b) noexcept {
- uint32_t a32, b32;
- memcpy(&a32, a, 4);
- memcpy(&b32, b, 4);
- return vreinterpretq_u8_u32(vsetq_lane_u32(b32, vdupq_n_u32(a32), 1));
-}
-
-simdjson_inline escape_vector escape_flags(escape_vector v) noexcept {
- uint8x16_t needs_escape = vceqq_u8(v, vdupq_n_u8(34)); // '"'
- needs_escape = vorrq_u8(needs_escape, vceqq_u8(v, vdupq_n_u8(92))); // '\\'
- return vorrq_u8(needs_escape, vcltq_u8(v, vdupq_n_u8(32)));
-}
-
-simdjson_inline bool escape_any(escape_vector flags) noexcept {
- return vmaxvq_u32(vreinterpretq_u32_u8(flags)) != 0;
-}
-
-// Four bits per byte rather than one: escape_block and the tail paths scale
-// their shifts by escape_mask_bits to match.
-simdjson_inline uint64_t escape_bitmask(escape_vector flags) noexcept {
- uint8x8_t narrowed = vshrn_n_u16(vreinterpretq_u16_u8(flags), 4);
- return vget_lane_u64(vreinterpret_u64_u8(narrowed), 0);
-}
-
-#endif // instruction set selection
-
-// The escape bitmask of a 16-byte block.
-simdjson_inline uint64_t escape_mask(escape_vector v) noexcept {
- return escape_bitmask(escape_flags(v));
-}
-
-// Copies n bytes with n < 16, using overlapping loads and stores. It never
-// reads more than n bytes from src, nor writes more than n bytes to dst.
-simdjson_inline void copy_lt16(char *dst, const uint8_t *src,
- size_t n) noexcept {
- if (n >= 8) {
- memcpy(dst, src, 8);
- memcpy(dst + n - 8, src + n - 8, 8);
- } else if (n >= 4) {
- memcpy(dst, src, 4);
- memcpy(dst + n - 4, src + n - 4, 4);
- } else if (n > 0) {
- dst[0] = char(src[0]);
- dst[n >> 1] = char(src[n >> 1]);
- dst[n - 1] = char(src[n - 1]);
- }
-}
-
-// Escapes the bytes of src in the range [i, blockend), given that m is the
-// (non-zero) escape mask for that range: the escape_mask_bits-wide lane of m
-// at byte k is non-zero when src[i + k] requires escaping. Returns the updated
-// output pointer.
-simdjson_never_inline char *escape_block(const uint8_t *src, char *out,
- size_t i, size_t blockend,
- uint64_t m) noexcept {
- constexpr uint64_t lane = (uint64_t(1) << escape_mask_bits) - 1;
-
- size_t pos = i; // first byte not yet copied
- while (m) {
- const size_t tz = trailing_zeroes(m);
- const size_t next = i + tz / escape_mask_bits;
- // Copy the run of safe bytes that precedes this escape.
- copy_lt16(out, src + pos, next - pos);
- out += next - pos;
- escape_json_char(char(src[next]), out);
- pos = next + 1;
- m &= ~(lane << tz);
- }
- // Copy whatever follows the last escape.
- copy_lt16(out, src + pos, blockend - pos);
- return out + (blockend - pos);
-}
-
-// Writes the escaped version of input to out, returning the number of bytes
-// written.
-simdjson_really_inline size_t write_string_escaped(const std::string_view input, char *out) {
- const size_t len = input.size();
- const uint8_t *src = reinterpret_cast<const uint8_t *>(input.data());
- const char *const initout = out;
-
- size_t i = 0;
-#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 && defined(__AVX2__)
- while (i + 32 <= len) {
- const __m256i word = _mm256_loadu_si256(reinterpret_cast<const __m256i *>(src + i));
- const __m256i flags = _mm256_or_si256(
- _mm256_or_si256(_mm256_cmpeq_epi8(word, _mm256_set1_epi8(34)), // '"'
- _mm256_cmpeq_epi8(word, _mm256_set1_epi8(92))), // '\\'
- _mm256_cmpeq_epi8(_mm256_subs_epu8(word, _mm256_set1_epi8(31)),
- _mm256_setzero_si256())); // control
- const uint32_t mask = uint32_t(_mm256_movemask_epi8(flags));
- if (simdjson_likely(mask == 0)) {
- _mm256_storeu_si256(reinterpret_cast<__m256i *>(out), word);
- out += 32;
- } else {
- for (size_t half = 0; half < 32; half += 16) {
- const uint64_t m = (mask >> half) & 0xFFFF;
- if (m == 0) {
- escape_store16(out, escape_load16(src + i + half));
- out += 16;
- } else {
- out = escape_block(src, out, i + half, i + half + 16, m);
- }
- }
- }
- i += 32;
- }
-#endif
- while (i + 16 <= len) {
- escape_vector word = escape_load16(src + i);
- escape_vector flags = escape_flags(word);
- if (simdjson_likely(!escape_any(flags))) {
- escape_store16(out, word);
- out += 16;
- } else {
- out = escape_block(src, out, i, i + 16, escape_bitmask(flags));
- }
- i += 16;
- }
- if (i < len) {
- const size_t rem = len - i;
- uint64_t m;
- if (len >= 16) {
- // The last 16 bytes of the input are in bounds. Bit k of that block's
- // mask belongs to input position len - 16 + k, so shift it down to align
- // bit 0 with position i.
- m = escape_mask(escape_load16(src + len - 16)) >>
- (escape_mask_bits * (16 - rem));
- } else if (len >= 8) {
- // Here i == 0 and rem == len. Two overlapping 8-byte loads cover
- // [0, 8) and [len - 8, len), which is the whole input since len < 16.
- uint64_t mm = escape_mask(escape_load8x2(src, src + len - 8));
- constexpr uint64_t low8 = (uint64_t(1) << (escape_mask_bits * 8)) - 1;
- m = (mm & low8) |
- ((mm >> (escape_mask_bits * 8)) << (escape_mask_bits * (len - 8)));
- } else if (len >= 4) {
- // Same idea with two overlapping 4-byte loads.
- uint64_t mm = escape_mask(escape_load4x2(src, src + len - 4));
- constexpr uint64_t low4 = (uint64_t(1) << (escape_mask_bits * 4)) - 1;
- m = (mm & low4) | (((mm >> (escape_mask_bits * 4)) & low4)
- << (escape_mask_bits * (len - 4)));
- } else {
- // Fewer than 4 bytes: at most three table lookups, no need for SIMD.
- for (size_t k = 0; k < len; k++) {
- uint8_t c = src[k];
- if (json_quotable_character[c]) {
- escape_json_char(char(c), out);
- } else {
- *out++ = char(c);
- }
- }
- return size_t(out - initout);
- }
- if (m == 0) {
- copy_lt16(out, src + i, rem);
- out += rem;
- } else {
- out = escape_block(src, out, i, len, m);
- }
- }
- return size_t(out - initout);
-}
-
-#else // SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-
-// Writes the escaped version of input to out, returning the number of bytes
-// written. Uses SIMD position finding to locate quotable characters efficiently.
-inline size_t write_string_escaped(const std::string_view input, char *out) {
- size_t mysize = input.size();
-
- // Use SIMD position finder directly - it returns mysize if no escape needed
- size_t location = find_next_json_quotable_character(input, 0);
- if (location == mysize) {
- // Fast path: no escaping needed
- memcpy(out, input.data(), input.size());
- return input.size();
- }
-
- const char *const initout = out;
- memcpy(out, input.data(), location);
- out += location;
- escape_json_char(input[location], out);
- location += 1;
- while (location < mysize) {
- size_t newlocation = find_next_json_quotable_character(input, location);
- memcpy(out, input.data() + location, newlocation - location);
- out += newlocation - location;
- location = newlocation;
- if (location == mysize) {
- break;
- }
- escape_json_char(input[location], out);
- location += 1;
- }
- return size_t(out - initout);
-}
-
-#endif // SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-#undef SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-
namespace internal {
// Sets the size of s to n without initializing the new bytes when the
@@ -65128,6 +62252,7 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin
/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */
/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_H */
/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */
+/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */
/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */
namespace simdjson {
@@ -65164,6 +62289,9 @@ struct has_custom_serialization : std::false_type {};
namespace lsx {
namespace builder {
+
+using simdjson::internal::escape_best::write_string_escaped;
+
/**
* A builder for JSON strings representing documents. This is a low-level
* builder that is not meant to be used directly by end-users. Though it
@@ -65510,6 +62638,7 @@ simdjson_warn_unused error_code to_json(const Z &z, std::string &s, size_t initi
/* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */
/* amalgamation skipped (editor-only): #include "simdjson/concepts.h" */
/* amalgamation skipped (editor-only): #include "simdjson/annotations.h" */
+/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */
/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */
#if SIMDJSON_STATIC_REFLECTION
@@ -65535,7 +62664,6 @@ namespace internal {
using simdjson::internal::write_uint_jeaiii;
simdjson_inline char *write_double(char *p, double v) noexcept;
} // namespace internal
-simdjson_really_inline size_t write_string_escaped(const std::string_view input, char *out);
SIMDJSON_PUSH_DISABLE_WARNINGS
SIMDJSON_DISABLE_GCC_WARNING(-Warray-bounds)
@@ -66556,619 +63684,13 @@ simdjson_warn_unused simdjson_result<std::string> extract_fractured_json(
/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_INL_H */
/* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */
/* amalgamation skipped (editor-only): #include "simdjson/internal/jeaiii.h" */
+/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */
/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */
-/*
- * Empirically, we have found that an inlined optimization is important for
- * performance. The following macros are not ideal. We should find a better
- * way to inline the code.
- */
-
-#if defined(__SSE2__) || defined(__x86_64__) || defined(__x86_64) || \
- (defined(_M_AMD64) || defined(_M_X64) || \
- (defined(_M_IX86_FP) && _M_IX86_FP == 2))
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_SSE2
-#define SIMDJSON_EXPERIMENTAL_HAS_SSE2 1
-#endif
-#endif
-
-#if defined(__aarch64__) || defined(_M_ARM64)
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_NEON
-#define SIMDJSON_EXPERIMENTAL_HAS_NEON 1
-#endif
-#endif
-#if defined(__loongarch_sx)
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_LSX
-#define SIMDJSON_EXPERIMENTAL_HAS_LSX 1
-#endif
-#endif
-#if defined(__loongarch_asx)
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_LASX
-#define SIMDJSON_EXPERIMENTAL_HAS_LASX 1
-#endif
-#endif
-#if defined(__riscv_v_intrinsic) && __riscv_v_intrinsic >= 11000 && \
- defined(__riscv_vector)
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_RVV
-#define SIMDJSON_EXPERIMENTAL_HAS_RVV 1
-#endif
-#endif
-#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) && defined(__POWER8_VECTOR__)
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64
-#define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1
-#endif
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_NEON
-#include <arm_neon.h>
-#ifdef _MSC_VER
-#include <intrin.h>
-#endif
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_SSE2
-#include <emmintrin.h>
-#if defined(__AVX2__)
-#include <immintrin.h>
-#endif
-#ifdef _MSC_VER
-#include <intrin.h>
-#endif
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_LSX
-#include <lsxintrin.h>
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_LASX
-#include <lasxintrin.h>
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_RVV
-#include <riscv_vector.h>
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_PPC64
-#include <altivec.h>
-#ifdef bool
-#undef bool
-#endif
-#ifdef vector
-#undef vector
-#endif
-#endif
-
-
namespace simdjson {
namespace lsx {
namespace builder {
-static SIMDJSON_CONSTEXPR_LAMBDA std::array<uint8_t, 256>
- json_quotable_character = {
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
-
-/**
-
-A possible SWAR implementation of has_json_escapable_byte. It is not used
-because it is slower than the current implementation. It is kept here for
-reference (to show that we tried it).
-
-inline bool has_json_escapable_byte(uint64_t x) {
- uint64_t is_ascii = 0x8080808080808080ULL & ~x;
- uint64_t xor2 = x ^ 0x0202020202020202ULL;
- uint64_t lt32_or_eq34 = xor2 - 0x2121212121212121ULL;
- uint64_t sub92 = x ^ 0x5C5C5C5C5C5C5C5CULL;
- uint64_t eq92 = (sub92 - 0x0101010101010101ULL);
- return ((lt32_or_eq34 | eq92) & is_ascii) != 0;
-}
-
-**/
-
-#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 || SIMDJSON_EXPERIMENTAL_HAS_NEON
-#define SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE 1
-#endif
-
-// With block escaping (SSE2/NEON), write_string_escaped scans the input itself.
-#if !SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-// Scalar fallback for finding next quotable character
-SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline size_t
-find_next_json_quotable_character_scalar(const std::string_view view,
- size_t location) noexcept {
- for (auto pos = view.begin() + location; pos != view.end(); ++pos) {
- if (json_quotable_character[static_cast<uint8_t>(*pos)]) {
- return pos - view.begin();
- }
- }
- return size_t(view.size());
-}
-
-// SIMD-accelerated position finding that directly locates the first quotable
-// character, combining detection and position extraction in a single pass to
-// minimize redundant work.
-#if SIMDJSON_EXPERIMENTAL_HAS_LASX
-simdjson_inline size_t
-find_next_json_quotable_character(const std::string_view view,
- size_t location) noexcept {
- const size_t len = view.size();
- const uint8_t *ptr =
- reinterpret_cast<const uint8_t *>(view.data()) + location;
- size_t remaining = len - location;
-
- // SIMD constants for characters requiring escape
- __m256i v34 = __lasx_xvreplgr2vr_b(34); // '"'
- __m256i v92 = __lasx_xvreplgr2vr_b(92); // '\\'
- __m256i v32 = __lasx_xvreplgr2vr_b(32); // control char threshold
-
- while (remaining >= 32) {
- __m256i word = __lasx_xvld(ptr, 0);
-
- // Check for quotable characters: '"', '\\', or control chars (< 32)
- __m256i needs_escape = __lasx_xvseq_b(word, v34);
- needs_escape = __lasx_xvor_v(needs_escape, __lasx_xvseq_b(word, v92));
- needs_escape = __lasx_xvor_v(needs_escape, __lasx_xvslt_bu(word, v32));
-
- if (!__lasx_xbz_v(needs_escape)) {
- // Found a quotable character - locate it via the four 64-bit lanes
- uint64_t lane0 = __lasx_xvpickve2gr_du(needs_escape, 0);
- uint64_t lane1 = __lasx_xvpickve2gr_du(needs_escape, 1);
- uint64_t lane2 = __lasx_xvpickve2gr_du(needs_escape, 2);
- uint64_t lane3 = __lasx_xvpickve2gr_du(needs_escape, 3);
- size_t offset = ptr - reinterpret_cast<const uint8_t *>(view.data());
- if (lane0 != 0) {
- return offset + trailing_zeroes(lane0) / 8;
- } else if (lane1 != 0) {
- return offset + 8 + trailing_zeroes(lane1) / 8;
- } else if (lane2 != 0) {
- return offset + 16 + trailing_zeroes(lane2) / 8;
- } else {
- return offset + 24 + trailing_zeroes(lane3) / 8;
- }
- }
- ptr += 32;
- remaining -= 32;
- }
- size_t current = len - remaining;
- return find_next_json_quotable_character_scalar(view, current);
-}
-#elif SIMDJSON_EXPERIMENTAL_HAS_LSX
-simdjson_inline size_t
-find_next_json_quotable_character(const std::string_view view,
- size_t location) noexcept {
- const size_t len = view.size();
- const uint8_t *ptr =
- reinterpret_cast<const uint8_t *>(view.data()) + location;
- size_t remaining = len - location;
-
- //SIMD constants for characters requiring escape
- __m128i v34 = __lsx_vreplgr2vr_b(34); // '"'
- __m128i v92 = __lsx_vreplgr2vr_b(92); // '\\'
- __m128i v32 = __lsx_vreplgr2vr_b(32); // control char threshold
-
- while (remaining >= 16){
- __m128i word = __lsx_vld(ptr, 0);
-
- //Check for the quotable characters: '"', '\\', or control char (<32)
- __m128i needs_escape = __lsx_vseq_b(word, v34);
- needs_escape = __lsx_vor_v(needs_escape, __lsx_vseq_b(word, v92));
- needs_escape = __lsx_vor_v(needs_escape, __lsx_vslt_bu(word, v32));
-
- if (!__lsx_bz_v(needs_escape)){
-
- //Found quotable character - extract exact byte position
- uint64_t lo = __lsx_vpickve2gr_du(needs_escape,0);
- uint64_t hi = __lsx_vpickve2gr_du(needs_escape,1);
- size_t offset = ptr - reinterpret_cast<const uint8_t *>(view.data());
- if ( lo != 0) {
- return offset + trailing_zeroes(lo) / 8;
- } else {
- return offset + 8 + trailing_zeroes(hi) / 8;
- }
- }
- ptr += 16;
- remaining -= 16;
- }
- size_t current = len - remaining;
- return find_next_json_quotable_character_scalar(view, current);
-}
-#elif SIMDJSON_EXPERIMENTAL_HAS_RVV
-simdjson_inline size_t
-find_next_json_quotable_character(const std::string_view view,
- size_t location) noexcept {
- const size_t len = view.size();
- const uint8_t *ptr =
- reinterpret_cast<const uint8_t *>(view.data()) + location;
- size_t remaining = len - location;
-
- while (remaining > 0) {
- size_t vl = __riscv_vsetvl_e8m1(remaining);
- vuint8m1_t word = __riscv_vle8_v_u8m1(ptr, vl);
-
- // Check for quotable characters: '"', '\\', or control chars (< 32)
- vbool8_t needs_escape = __riscv_vmseq(word, (uint8_t)34, vl);
- needs_escape = __riscv_vmor(needs_escape,
- __riscv_vmseq(word, (uint8_t)92, vl), vl);
- needs_escape = __riscv_vmor(needs_escape,
- __riscv_vmsltu(word, (uint8_t)32, vl), vl);
-
- long first = __riscv_vfirst(needs_escape, vl);
- if (first >= 0) {
- size_t offset = ptr - reinterpret_cast<const uint8_t *>(view.data());
- return offset + first;
- }
- ptr += vl;
- remaining -= vl;
- }
-
- return len;
-}
-#elif SIMDJSON_EXPERIMENTAL_HAS_PPC64
-simdjson_inline size_t
-find_next_json_quotable_character(const std::string_view view,
- size_t location) noexcept {
- const size_t len = view.size();
- const uint8_t *ptr =
- reinterpret_cast<const uint8_t *>(view.data()) + location;
- size_t remaining = len - location;
-
- // SIMD constants for characters requiring escape
- __vector unsigned char v34 = vec_splats((unsigned char)34); // '"'
- __vector unsigned char v92 = vec_splats((unsigned char)92); // '\\'
- __vector unsigned char v32 = vec_splats((unsigned char)32); // control char threshold
-
- // Bitmask for vec_vbpermq to extract one bit per byte
- const __vector unsigned char perm_mask = {0x78, 0x70, 0x68, 0x60, 0x58, 0x50,
- 0x48, 0x40, 0x38, 0x30, 0x28, 0x20,
- 0x18, 0x10, 0x08, 0x00};
-
- while (remaining >= 16) {
- __vector unsigned char word =
- vec_vsx_ld(0, reinterpret_cast<const unsigned char *>(ptr));
-
- // Check for quotable characters: '"', '\\', or control chars (< 32)
- __vector unsigned char needs_escape =
- (__vector unsigned char)vec_cmpeq(word, v34);
- needs_escape = vec_or(needs_escape,
- (__vector unsigned char)vec_cmpeq(word, v92));
- needs_escape = vec_or(needs_escape,
- (__vector unsigned char)vec_cmplt(word, v32));
-
- __vector unsigned long long result =
- (__vector unsigned long long)vec_vbpermq(needs_escape, perm_mask);
-#ifdef __LITTLE_ENDIAN__
- unsigned int mask = static_cast<unsigned int>(result[1]);
-#else
- unsigned int mask = static_cast<unsigned int>(result[0]);
-#endif
- if (mask != 0) {
- size_t offset = ptr - reinterpret_cast<const uint8_t *>(view.data());
- return offset + __builtin_ctz(mask);
- }
- ptr += 16;
- remaining -= 16;
- }
-
- // Scalar fallback for remaining bytes
- size_t current = len - remaining;
- return find_next_json_quotable_character_scalar(view, current);
-}
-#else
-SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline size_t
-find_next_json_quotable_character(const std::string_view view,
- size_t location) noexcept {
- return find_next_json_quotable_character_scalar(view, location);
-}
-#endif
-#endif // !SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-
-SIMDJSON_CONSTEXPR_LAMBDA static std::string_view control_chars[] = {
- "\\u0000", "\\u0001", "\\u0002", "\\u0003", "\\u0004", "\\u0005", "\\u0006",
- "\\u0007", "\\b", "\\t", "\\n", "\\u000b", "\\f", "\\r",
- "\\u000e", "\\u000f", "\\u0010", "\\u0011", "\\u0012", "\\u0013", "\\u0014",
- "\\u0015", "\\u0016", "\\u0017", "\\u0018", "\\u0019", "\\u001a", "\\u001b",
- "\\u001c", "\\u001d", "\\u001e", "\\u001f"};
-
-// All Unicode characters may be placed within the quotation marks, except for
-// the characters that MUST be escaped: quotation mark, reverse solidus, and the
-// control characters (U+0000 through U+001F). There are two-character sequence
-// escape representations of some popular characters:
-// \", \\, \b, \f, \n, \r, \t.
-SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline void escape_json_char(char c, char *&out) {
- if (c == '"') {
- memcpy(out, "\\\"", 2);
- out += 2;
- } else if (c == '\\') {
- memcpy(out, "\\\\", 2);
- out += 2;
- } else {
- std::string_view v = control_chars[uint8_t(c)];
- memcpy(out, v.data(), v.size());
- out += v.size();
- }
-}
-
-#if SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-
-#if SIMDJSON_EXPERIMENTAL_HAS_SSE2
-
-using escape_vector = __m128i;
-// Mask bits that each input byte contributes to escape_bitmask().
-static constexpr unsigned escape_mask_bits = 1;
-
-simdjson_inline escape_vector escape_load16(const uint8_t *p) noexcept {
- return _mm_loadu_si128(reinterpret_cast<const __m128i *>(p));
-}
-
-simdjson_inline void escape_store16(char *out, escape_vector v) noexcept {
- _mm_storeu_si128(reinterpret_cast<__m128i *>(out), v);
-}
-
-// Builds a vector whose bytes 0..7 come from a and bytes 8..15 from b.
-simdjson_inline escape_vector escape_load8x2(const uint8_t *a,
- const uint8_t *b) noexcept {
- return _mm_unpacklo_epi64(
- _mm_loadl_epi64(reinterpret_cast<const __m128i *>(a)),
- _mm_loadl_epi64(reinterpret_cast<const __m128i *>(b)));
-}
-
-// Builds a vector whose bytes 0..3 come from a and bytes 4..7 from b. The
-// upper half is unspecified; callers only look at the low eight bits.
-simdjson_inline escape_vector escape_load4x2(const uint8_t *a,
- const uint8_t *b) noexcept {
- int32_t a32, b32;
- memcpy(&a32, a, 4);
- memcpy(&b32, b, 4);
- return _mm_unpacklo_epi32(_mm_cvtsi32_si128(a32), _mm_cvtsi32_si128(b32));
-}
-
-// Sets every bit of byte k when byte k of v is a quotable character ('"',
-// '\\' or a control character).
-simdjson_inline escape_vector escape_flags(escape_vector v) noexcept {
- const __m128i v34 = _mm_set1_epi8(34); // '"'
- const __m128i v92 = _mm_set1_epi8(92); // '\\'
- const __m128i v31 = _mm_set1_epi8(31); // for control char detection
- __m128i needs_escape = _mm_cmpeq_epi8(v, v34);
- needs_escape = _mm_or_si128(needs_escape, _mm_cmpeq_epi8(v, v92));
- return _mm_or_si128(
- needs_escape, _mm_cmpeq_epi8(_mm_subs_epu8(v, v31), _mm_setzero_si128()));
-}
-
-// True when any byte needs escaping. Kept separate from escape_bitmask
-// because some instruction sets can answer it without leaving the vector
-// register file; on SSE2 the compiler folds the two together.
-simdjson_inline bool escape_any(escape_vector flags) noexcept {
- return _mm_movemask_epi8(flags) != 0;
-}
-
-// A 16-bit mask with bit k set when byte k needed escaping.
-simdjson_inline uint64_t escape_bitmask(escape_vector flags) noexcept {
- return uint64_t(uint32_t(_mm_movemask_epi8(flags)));
-}
-
-#else // SIMDJSON_EXPERIMENTAL_HAS_NEON
-
-using escape_vector = uint8x16_t;
-static constexpr unsigned escape_mask_bits = 4;
-
-simdjson_inline escape_vector escape_load16(const uint8_t *p) noexcept {
- return vld1q_u8(p);
-}
-
-simdjson_inline void escape_store16(char *out, escape_vector v) noexcept {
- vst1q_u8(reinterpret_cast<uint8_t *>(out), v);
-}
-
-simdjson_inline escape_vector escape_load8x2(const uint8_t *a,
- const uint8_t *b) noexcept {
- uint64_t a64, b64;
- memcpy(&a64, a, 8);
- memcpy(&b64, b, 8);
- return vreinterpretq_u8_u64(vsetq_lane_u64(b64, vdupq_n_u64(a64), 1));
-}
-
-simdjson_inline escape_vector escape_load4x2(const uint8_t *a,
- const uint8_t *b) noexcept {
- uint32_t a32, b32;
- memcpy(&a32, a, 4);
- memcpy(&b32, b, 4);
- return vreinterpretq_u8_u32(vsetq_lane_u32(b32, vdupq_n_u32(a32), 1));
-}
-
-simdjson_inline escape_vector escape_flags(escape_vector v) noexcept {
- uint8x16_t needs_escape = vceqq_u8(v, vdupq_n_u8(34)); // '"'
- needs_escape = vorrq_u8(needs_escape, vceqq_u8(v, vdupq_n_u8(92))); // '\\'
- return vorrq_u8(needs_escape, vcltq_u8(v, vdupq_n_u8(32)));
-}
-
-simdjson_inline bool escape_any(escape_vector flags) noexcept {
- return vmaxvq_u32(vreinterpretq_u32_u8(flags)) != 0;
-}
-
-// Four bits per byte rather than one: escape_block and the tail paths scale
-// their shifts by escape_mask_bits to match.
-simdjson_inline uint64_t escape_bitmask(escape_vector flags) noexcept {
- uint8x8_t narrowed = vshrn_n_u16(vreinterpretq_u16_u8(flags), 4);
- return vget_lane_u64(vreinterpret_u64_u8(narrowed), 0);
-}
-
-#endif // instruction set selection
-
-// The escape bitmask of a 16-byte block.
-simdjson_inline uint64_t escape_mask(escape_vector v) noexcept {
- return escape_bitmask(escape_flags(v));
-}
-
-// Copies n bytes with n < 16, using overlapping loads and stores. It never
-// reads more than n bytes from src, nor writes more than n bytes to dst.
-simdjson_inline void copy_lt16(char *dst, const uint8_t *src,
- size_t n) noexcept {
- if (n >= 8) {
- memcpy(dst, src, 8);
- memcpy(dst + n - 8, src + n - 8, 8);
- } else if (n >= 4) {
- memcpy(dst, src, 4);
- memcpy(dst + n - 4, src + n - 4, 4);
- } else if (n > 0) {
- dst[0] = char(src[0]);
- dst[n >> 1] = char(src[n >> 1]);
- dst[n - 1] = char(src[n - 1]);
- }
-}
-
-// Escapes the bytes of src in the range [i, blockend), given that m is the
-// (non-zero) escape mask for that range: the escape_mask_bits-wide lane of m
-// at byte k is non-zero when src[i + k] requires escaping. Returns the updated
-// output pointer.
-simdjson_never_inline char *escape_block(const uint8_t *src, char *out,
- size_t i, size_t blockend,
- uint64_t m) noexcept {
- constexpr uint64_t lane = (uint64_t(1) << escape_mask_bits) - 1;
-
- size_t pos = i; // first byte not yet copied
- while (m) {
- const size_t tz = trailing_zeroes(m);
- const size_t next = i + tz / escape_mask_bits;
- // Copy the run of safe bytes that precedes this escape.
- copy_lt16(out, src + pos, next - pos);
- out += next - pos;
- escape_json_char(char(src[next]), out);
- pos = next + 1;
- m &= ~(lane << tz);
- }
- // Copy whatever follows the last escape.
- copy_lt16(out, src + pos, blockend - pos);
- return out + (blockend - pos);
-}
-
-// Writes the escaped version of input to out, returning the number of bytes
-// written.
-simdjson_really_inline size_t write_string_escaped(const std::string_view input, char *out) {
- const size_t len = input.size();
- const uint8_t *src = reinterpret_cast<const uint8_t *>(input.data());
- const char *const initout = out;
-
- size_t i = 0;
-#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 && defined(__AVX2__)
- while (i + 32 <= len) {
- const __m256i word = _mm256_loadu_si256(reinterpret_cast<const __m256i *>(src + i));
- const __m256i flags = _mm256_or_si256(
- _mm256_or_si256(_mm256_cmpeq_epi8(word, _mm256_set1_epi8(34)), // '"'
- _mm256_cmpeq_epi8(word, _mm256_set1_epi8(92))), // '\\'
- _mm256_cmpeq_epi8(_mm256_subs_epu8(word, _mm256_set1_epi8(31)),
- _mm256_setzero_si256())); // control
- const uint32_t mask = uint32_t(_mm256_movemask_epi8(flags));
- if (simdjson_likely(mask == 0)) {
- _mm256_storeu_si256(reinterpret_cast<__m256i *>(out), word);
- out += 32;
- } else {
- for (size_t half = 0; half < 32; half += 16) {
- const uint64_t m = (mask >> half) & 0xFFFF;
- if (m == 0) {
- escape_store16(out, escape_load16(src + i + half));
- out += 16;
- } else {
- out = escape_block(src, out, i + half, i + half + 16, m);
- }
- }
- }
- i += 32;
- }
-#endif
- while (i + 16 <= len) {
- escape_vector word = escape_load16(src + i);
- escape_vector flags = escape_flags(word);
- if (simdjson_likely(!escape_any(flags))) {
- escape_store16(out, word);
- out += 16;
- } else {
- out = escape_block(src, out, i, i + 16, escape_bitmask(flags));
- }
- i += 16;
- }
- if (i < len) {
- const size_t rem = len - i;
- uint64_t m;
- if (len >= 16) {
- // The last 16 bytes of the input are in bounds. Bit k of that block's
- // mask belongs to input position len - 16 + k, so shift it down to align
- // bit 0 with position i.
- m = escape_mask(escape_load16(src + len - 16)) >>
- (escape_mask_bits * (16 - rem));
- } else if (len >= 8) {
- // Here i == 0 and rem == len. Two overlapping 8-byte loads cover
- // [0, 8) and [len - 8, len), which is the whole input since len < 16.
- uint64_t mm = escape_mask(escape_load8x2(src, src + len - 8));
- constexpr uint64_t low8 = (uint64_t(1) << (escape_mask_bits * 8)) - 1;
- m = (mm & low8) |
- ((mm >> (escape_mask_bits * 8)) << (escape_mask_bits * (len - 8)));
- } else if (len >= 4) {
- // Same idea with two overlapping 4-byte loads.
- uint64_t mm = escape_mask(escape_load4x2(src, src + len - 4));
- constexpr uint64_t low4 = (uint64_t(1) << (escape_mask_bits * 4)) - 1;
- m = (mm & low4) | (((mm >> (escape_mask_bits * 4)) & low4)
- << (escape_mask_bits * (len - 4)));
- } else {
- // Fewer than 4 bytes: at most three table lookups, no need for SIMD.
- for (size_t k = 0; k < len; k++) {
- uint8_t c = src[k];
- if (json_quotable_character[c]) {
- escape_json_char(char(c), out);
- } else {
- *out++ = char(c);
- }
- }
- return size_t(out - initout);
- }
- if (m == 0) {
- copy_lt16(out, src + i, rem);
- out += rem;
- } else {
- out = escape_block(src, out, i, len, m);
- }
- }
- return size_t(out - initout);
-}
-
-#else // SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-
-// Writes the escaped version of input to out, returning the number of bytes
-// written. Uses SIMD position finding to locate quotable characters efficiently.
-inline size_t write_string_escaped(const std::string_view input, char *out) {
- size_t mysize = input.size();
-
- // Use SIMD position finder directly - it returns mysize if no escape needed
- size_t location = find_next_json_quotable_character(input, 0);
- if (location == mysize) {
- // Fast path: no escaping needed
- memcpy(out, input.data(), input.size());
- return input.size();
- }
-
- const char *const initout = out;
- memcpy(out, input.data(), location);
- out += location;
- escape_json_char(input[location], out);
- location += 1;
- while (location < mysize) {
- size_t newlocation = find_next_json_quotable_character(input, location);
- memcpy(out, input.data() + location, newlocation - location);
- out += newlocation - location;
- location = newlocation;
- if (location == mysize) {
- break;
- }
- escape_json_char(input[location], out);
- location += 1;
- }
- return size_t(out - initout);
-}
-
-#endif // SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-#undef SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-
namespace internal {
// Sets the size of s to n without initializing the new bytes when the
@@ -68428,6 +64950,7 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin
/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */
/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_H */
/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */
+/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */
/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */
namespace simdjson {
@@ -68464,6 +64987,9 @@ struct has_custom_serialization : std::false_type {};
namespace lasx {
namespace builder {
+
+using simdjson::internal::escape_best::write_string_escaped;
+
/**
* A builder for JSON strings representing documents. This is a low-level
* builder that is not meant to be used directly by end-users. Though it
@@ -68810,6 +65336,7 @@ simdjson_warn_unused error_code to_json(const Z &z, std::string &s, size_t initi
/* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */
/* amalgamation skipped (editor-only): #include "simdjson/concepts.h" */
/* amalgamation skipped (editor-only): #include "simdjson/annotations.h" */
+/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */
/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */
#if SIMDJSON_STATIC_REFLECTION
@@ -68835,7 +65362,6 @@ namespace internal {
using simdjson::internal::write_uint_jeaiii;
simdjson_inline char *write_double(char *p, double v) noexcept;
} // namespace internal
-simdjson_really_inline size_t write_string_escaped(const std::string_view input, char *out);
SIMDJSON_PUSH_DISABLE_WARNINGS
SIMDJSON_DISABLE_GCC_WARNING(-Warray-bounds)
@@ -69856,619 +66382,13 @@ simdjson_warn_unused simdjson_result<std::string> extract_fractured_json(
/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_INL_H */
/* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */
/* amalgamation skipped (editor-only): #include "simdjson/internal/jeaiii.h" */
+/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */
/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */
-/*
- * Empirically, we have found that an inlined optimization is important for
- * performance. The following macros are not ideal. We should find a better
- * way to inline the code.
- */
-
-#if defined(__SSE2__) || defined(__x86_64__) || defined(__x86_64) || \
- (defined(_M_AMD64) || defined(_M_X64) || \
- (defined(_M_IX86_FP) && _M_IX86_FP == 2))
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_SSE2
-#define SIMDJSON_EXPERIMENTAL_HAS_SSE2 1
-#endif
-#endif
-
-#if defined(__aarch64__) || defined(_M_ARM64)
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_NEON
-#define SIMDJSON_EXPERIMENTAL_HAS_NEON 1
-#endif
-#endif
-#if defined(__loongarch_sx)
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_LSX
-#define SIMDJSON_EXPERIMENTAL_HAS_LSX 1
-#endif
-#endif
-#if defined(__loongarch_asx)
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_LASX
-#define SIMDJSON_EXPERIMENTAL_HAS_LASX 1
-#endif
-#endif
-#if defined(__riscv_v_intrinsic) && __riscv_v_intrinsic >= 11000 && \
- defined(__riscv_vector)
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_RVV
-#define SIMDJSON_EXPERIMENTAL_HAS_RVV 1
-#endif
-#endif
-#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) && defined(__POWER8_VECTOR__)
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64
-#define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1
-#endif
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_NEON
-#include <arm_neon.h>
-#ifdef _MSC_VER
-#include <intrin.h>
-#endif
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_SSE2
-#include <emmintrin.h>
-#if defined(__AVX2__)
-#include <immintrin.h>
-#endif
-#ifdef _MSC_VER
-#include <intrin.h>
-#endif
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_LSX
-#include <lsxintrin.h>
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_LASX
-#include <lasxintrin.h>
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_RVV
-#include <riscv_vector.h>
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_PPC64
-#include <altivec.h>
-#ifdef bool
-#undef bool
-#endif
-#ifdef vector
-#undef vector
-#endif
-#endif
-
-
namespace simdjson {
namespace lasx {
namespace builder {
-static SIMDJSON_CONSTEXPR_LAMBDA std::array<uint8_t, 256>
- json_quotable_character = {
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
-
-/**
-
-A possible SWAR implementation of has_json_escapable_byte. It is not used
-because it is slower than the current implementation. It is kept here for
-reference (to show that we tried it).
-
-inline bool has_json_escapable_byte(uint64_t x) {
- uint64_t is_ascii = 0x8080808080808080ULL & ~x;
- uint64_t xor2 = x ^ 0x0202020202020202ULL;
- uint64_t lt32_or_eq34 = xor2 - 0x2121212121212121ULL;
- uint64_t sub92 = x ^ 0x5C5C5C5C5C5C5C5CULL;
- uint64_t eq92 = (sub92 - 0x0101010101010101ULL);
- return ((lt32_or_eq34 | eq92) & is_ascii) != 0;
-}
-
-**/
-
-#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 || SIMDJSON_EXPERIMENTAL_HAS_NEON
-#define SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE 1
-#endif
-
-// With block escaping (SSE2/NEON), write_string_escaped scans the input itself.
-#if !SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-// Scalar fallback for finding next quotable character
-SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline size_t
-find_next_json_quotable_character_scalar(const std::string_view view,
- size_t location) noexcept {
- for (auto pos = view.begin() + location; pos != view.end(); ++pos) {
- if (json_quotable_character[static_cast<uint8_t>(*pos)]) {
- return pos - view.begin();
- }
- }
- return size_t(view.size());
-}
-
-// SIMD-accelerated position finding that directly locates the first quotable
-// character, combining detection and position extraction in a single pass to
-// minimize redundant work.
-#if SIMDJSON_EXPERIMENTAL_HAS_LASX
-simdjson_inline size_t
-find_next_json_quotable_character(const std::string_view view,
- size_t location) noexcept {
- const size_t len = view.size();
- const uint8_t *ptr =
- reinterpret_cast<const uint8_t *>(view.data()) + location;
- size_t remaining = len - location;
-
- // SIMD constants for characters requiring escape
- __m256i v34 = __lasx_xvreplgr2vr_b(34); // '"'
- __m256i v92 = __lasx_xvreplgr2vr_b(92); // '\\'
- __m256i v32 = __lasx_xvreplgr2vr_b(32); // control char threshold
-
- while (remaining >= 32) {
- __m256i word = __lasx_xvld(ptr, 0);
-
- // Check for quotable characters: '"', '\\', or control chars (< 32)
- __m256i needs_escape = __lasx_xvseq_b(word, v34);
- needs_escape = __lasx_xvor_v(needs_escape, __lasx_xvseq_b(word, v92));
- needs_escape = __lasx_xvor_v(needs_escape, __lasx_xvslt_bu(word, v32));
-
- if (!__lasx_xbz_v(needs_escape)) {
- // Found a quotable character - locate it via the four 64-bit lanes
- uint64_t lane0 = __lasx_xvpickve2gr_du(needs_escape, 0);
- uint64_t lane1 = __lasx_xvpickve2gr_du(needs_escape, 1);
- uint64_t lane2 = __lasx_xvpickve2gr_du(needs_escape, 2);
- uint64_t lane3 = __lasx_xvpickve2gr_du(needs_escape, 3);
- size_t offset = ptr - reinterpret_cast<const uint8_t *>(view.data());
- if (lane0 != 0) {
- return offset + trailing_zeroes(lane0) / 8;
- } else if (lane1 != 0) {
- return offset + 8 + trailing_zeroes(lane1) / 8;
- } else if (lane2 != 0) {
- return offset + 16 + trailing_zeroes(lane2) / 8;
- } else {
- return offset + 24 + trailing_zeroes(lane3) / 8;
- }
- }
- ptr += 32;
- remaining -= 32;
- }
- size_t current = len - remaining;
- return find_next_json_quotable_character_scalar(view, current);
-}
-#elif SIMDJSON_EXPERIMENTAL_HAS_LSX
-simdjson_inline size_t
-find_next_json_quotable_character(const std::string_view view,
- size_t location) noexcept {
- const size_t len = view.size();
- const uint8_t *ptr =
- reinterpret_cast<const uint8_t *>(view.data()) + location;
- size_t remaining = len - location;
-
- //SIMD constants for characters requiring escape
- __m128i v34 = __lsx_vreplgr2vr_b(34); // '"'
- __m128i v92 = __lsx_vreplgr2vr_b(92); // '\\'
- __m128i v32 = __lsx_vreplgr2vr_b(32); // control char threshold
-
- while (remaining >= 16){
- __m128i word = __lsx_vld(ptr, 0);
-
- //Check for the quotable characters: '"', '\\', or control char (<32)
- __m128i needs_escape = __lsx_vseq_b(word, v34);
- needs_escape = __lsx_vor_v(needs_escape, __lsx_vseq_b(word, v92));
- needs_escape = __lsx_vor_v(needs_escape, __lsx_vslt_bu(word, v32));
-
- if (!__lsx_bz_v(needs_escape)){
-
- //Found quotable character - extract exact byte position
- uint64_t lo = __lsx_vpickve2gr_du(needs_escape,0);
- uint64_t hi = __lsx_vpickve2gr_du(needs_escape,1);
- size_t offset = ptr - reinterpret_cast<const uint8_t *>(view.data());
- if ( lo != 0) {
- return offset + trailing_zeroes(lo) / 8;
- } else {
- return offset + 8 + trailing_zeroes(hi) / 8;
- }
- }
- ptr += 16;
- remaining -= 16;
- }
- size_t current = len - remaining;
- return find_next_json_quotable_character_scalar(view, current);
-}
-#elif SIMDJSON_EXPERIMENTAL_HAS_RVV
-simdjson_inline size_t
-find_next_json_quotable_character(const std::string_view view,
- size_t location) noexcept {
- const size_t len = view.size();
- const uint8_t *ptr =
- reinterpret_cast<const uint8_t *>(view.data()) + location;
- size_t remaining = len - location;
-
- while (remaining > 0) {
- size_t vl = __riscv_vsetvl_e8m1(remaining);
- vuint8m1_t word = __riscv_vle8_v_u8m1(ptr, vl);
-
- // Check for quotable characters: '"', '\\', or control chars (< 32)
- vbool8_t needs_escape = __riscv_vmseq(word, (uint8_t)34, vl);
- needs_escape = __riscv_vmor(needs_escape,
- __riscv_vmseq(word, (uint8_t)92, vl), vl);
- needs_escape = __riscv_vmor(needs_escape,
- __riscv_vmsltu(word, (uint8_t)32, vl), vl);
-
- long first = __riscv_vfirst(needs_escape, vl);
- if (first >= 0) {
- size_t offset = ptr - reinterpret_cast<const uint8_t *>(view.data());
- return offset + first;
- }
- ptr += vl;
- remaining -= vl;
- }
-
- return len;
-}
-#elif SIMDJSON_EXPERIMENTAL_HAS_PPC64
-simdjson_inline size_t
-find_next_json_quotable_character(const std::string_view view,
- size_t location) noexcept {
- const size_t len = view.size();
- const uint8_t *ptr =
- reinterpret_cast<const uint8_t *>(view.data()) + location;
- size_t remaining = len - location;
-
- // SIMD constants for characters requiring escape
- __vector unsigned char v34 = vec_splats((unsigned char)34); // '"'
- __vector unsigned char v92 = vec_splats((unsigned char)92); // '\\'
- __vector unsigned char v32 = vec_splats((unsigned char)32); // control char threshold
-
- // Bitmask for vec_vbpermq to extract one bit per byte
- const __vector unsigned char perm_mask = {0x78, 0x70, 0x68, 0x60, 0x58, 0x50,
- 0x48, 0x40, 0x38, 0x30, 0x28, 0x20,
- 0x18, 0x10, 0x08, 0x00};
-
- while (remaining >= 16) {
- __vector unsigned char word =
- vec_vsx_ld(0, reinterpret_cast<const unsigned char *>(ptr));
-
- // Check for quotable characters: '"', '\\', or control chars (< 32)
- __vector unsigned char needs_escape =
- (__vector unsigned char)vec_cmpeq(word, v34);
- needs_escape = vec_or(needs_escape,
- (__vector unsigned char)vec_cmpeq(word, v92));
- needs_escape = vec_or(needs_escape,
- (__vector unsigned char)vec_cmplt(word, v32));
-
- __vector unsigned long long result =
- (__vector unsigned long long)vec_vbpermq(needs_escape, perm_mask);
-#ifdef __LITTLE_ENDIAN__
- unsigned int mask = static_cast<unsigned int>(result[1]);
-#else
- unsigned int mask = static_cast<unsigned int>(result[0]);
-#endif
- if (mask != 0) {
- size_t offset = ptr - reinterpret_cast<const uint8_t *>(view.data());
- return offset + __builtin_ctz(mask);
- }
- ptr += 16;
- remaining -= 16;
- }
-
- // Scalar fallback for remaining bytes
- size_t current = len - remaining;
- return find_next_json_quotable_character_scalar(view, current);
-}
-#else
-SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline size_t
-find_next_json_quotable_character(const std::string_view view,
- size_t location) noexcept {
- return find_next_json_quotable_character_scalar(view, location);
-}
-#endif
-#endif // !SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-
-SIMDJSON_CONSTEXPR_LAMBDA static std::string_view control_chars[] = {
- "\\u0000", "\\u0001", "\\u0002", "\\u0003", "\\u0004", "\\u0005", "\\u0006",
- "\\u0007", "\\b", "\\t", "\\n", "\\u000b", "\\f", "\\r",
- "\\u000e", "\\u000f", "\\u0010", "\\u0011", "\\u0012", "\\u0013", "\\u0014",
- "\\u0015", "\\u0016", "\\u0017", "\\u0018", "\\u0019", "\\u001a", "\\u001b",
- "\\u001c", "\\u001d", "\\u001e", "\\u001f"};
-
-// All Unicode characters may be placed within the quotation marks, except for
-// the characters that MUST be escaped: quotation mark, reverse solidus, and the
-// control characters (U+0000 through U+001F). There are two-character sequence
-// escape representations of some popular characters:
-// \", \\, \b, \f, \n, \r, \t.
-SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline void escape_json_char(char c, char *&out) {
- if (c == '"') {
- memcpy(out, "\\\"", 2);
- out += 2;
- } else if (c == '\\') {
- memcpy(out, "\\\\", 2);
- out += 2;
- } else {
- std::string_view v = control_chars[uint8_t(c)];
- memcpy(out, v.data(), v.size());
- out += v.size();
- }
-}
-
-#if SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-
-#if SIMDJSON_EXPERIMENTAL_HAS_SSE2
-
-using escape_vector = __m128i;
-// Mask bits that each input byte contributes to escape_bitmask().
-static constexpr unsigned escape_mask_bits = 1;
-
-simdjson_inline escape_vector escape_load16(const uint8_t *p) noexcept {
- return _mm_loadu_si128(reinterpret_cast<const __m128i *>(p));
-}
-
-simdjson_inline void escape_store16(char *out, escape_vector v) noexcept {
- _mm_storeu_si128(reinterpret_cast<__m128i *>(out), v);
-}
-
-// Builds a vector whose bytes 0..7 come from a and bytes 8..15 from b.
-simdjson_inline escape_vector escape_load8x2(const uint8_t *a,
- const uint8_t *b) noexcept {
- return _mm_unpacklo_epi64(
- _mm_loadl_epi64(reinterpret_cast<const __m128i *>(a)),
- _mm_loadl_epi64(reinterpret_cast<const __m128i *>(b)));
-}
-
-// Builds a vector whose bytes 0..3 come from a and bytes 4..7 from b. The
-// upper half is unspecified; callers only look at the low eight bits.
-simdjson_inline escape_vector escape_load4x2(const uint8_t *a,
- const uint8_t *b) noexcept {
- int32_t a32, b32;
- memcpy(&a32, a, 4);
- memcpy(&b32, b, 4);
- return _mm_unpacklo_epi32(_mm_cvtsi32_si128(a32), _mm_cvtsi32_si128(b32));
-}
-
-// Sets every bit of byte k when byte k of v is a quotable character ('"',
-// '\\' or a control character).
-simdjson_inline escape_vector escape_flags(escape_vector v) noexcept {
- const __m128i v34 = _mm_set1_epi8(34); // '"'
- const __m128i v92 = _mm_set1_epi8(92); // '\\'
- const __m128i v31 = _mm_set1_epi8(31); // for control char detection
- __m128i needs_escape = _mm_cmpeq_epi8(v, v34);
- needs_escape = _mm_or_si128(needs_escape, _mm_cmpeq_epi8(v, v92));
- return _mm_or_si128(
- needs_escape, _mm_cmpeq_epi8(_mm_subs_epu8(v, v31), _mm_setzero_si128()));
-}
-
-// True when any byte needs escaping. Kept separate from escape_bitmask
-// because some instruction sets can answer it without leaving the vector
-// register file; on SSE2 the compiler folds the two together.
-simdjson_inline bool escape_any(escape_vector flags) noexcept {
- return _mm_movemask_epi8(flags) != 0;
-}
-
-// A 16-bit mask with bit k set when byte k needed escaping.
-simdjson_inline uint64_t escape_bitmask(escape_vector flags) noexcept {
- return uint64_t(uint32_t(_mm_movemask_epi8(flags)));
-}
-
-#else // SIMDJSON_EXPERIMENTAL_HAS_NEON
-
-using escape_vector = uint8x16_t;
-static constexpr unsigned escape_mask_bits = 4;
-
-simdjson_inline escape_vector escape_load16(const uint8_t *p) noexcept {
- return vld1q_u8(p);
-}
-
-simdjson_inline void escape_store16(char *out, escape_vector v) noexcept {
- vst1q_u8(reinterpret_cast<uint8_t *>(out), v);
-}
-
-simdjson_inline escape_vector escape_load8x2(const uint8_t *a,
- const uint8_t *b) noexcept {
- uint64_t a64, b64;
- memcpy(&a64, a, 8);
- memcpy(&b64, b, 8);
- return vreinterpretq_u8_u64(vsetq_lane_u64(b64, vdupq_n_u64(a64), 1));
-}
-
-simdjson_inline escape_vector escape_load4x2(const uint8_t *a,
- const uint8_t *b) noexcept {
- uint32_t a32, b32;
- memcpy(&a32, a, 4);
- memcpy(&b32, b, 4);
- return vreinterpretq_u8_u32(vsetq_lane_u32(b32, vdupq_n_u32(a32), 1));
-}
-
-simdjson_inline escape_vector escape_flags(escape_vector v) noexcept {
- uint8x16_t needs_escape = vceqq_u8(v, vdupq_n_u8(34)); // '"'
- needs_escape = vorrq_u8(needs_escape, vceqq_u8(v, vdupq_n_u8(92))); // '\\'
- return vorrq_u8(needs_escape, vcltq_u8(v, vdupq_n_u8(32)));
-}
-
-simdjson_inline bool escape_any(escape_vector flags) noexcept {
- return vmaxvq_u32(vreinterpretq_u32_u8(flags)) != 0;
-}
-
-// Four bits per byte rather than one: escape_block and the tail paths scale
-// their shifts by escape_mask_bits to match.
-simdjson_inline uint64_t escape_bitmask(escape_vector flags) noexcept {
- uint8x8_t narrowed = vshrn_n_u16(vreinterpretq_u16_u8(flags), 4);
- return vget_lane_u64(vreinterpret_u64_u8(narrowed), 0);
-}
-
-#endif // instruction set selection
-
-// The escape bitmask of a 16-byte block.
-simdjson_inline uint64_t escape_mask(escape_vector v) noexcept {
- return escape_bitmask(escape_flags(v));
-}
-
-// Copies n bytes with n < 16, using overlapping loads and stores. It never
-// reads more than n bytes from src, nor writes more than n bytes to dst.
-simdjson_inline void copy_lt16(char *dst, const uint8_t *src,
- size_t n) noexcept {
- if (n >= 8) {
- memcpy(dst, src, 8);
- memcpy(dst + n - 8, src + n - 8, 8);
- } else if (n >= 4) {
- memcpy(dst, src, 4);
- memcpy(dst + n - 4, src + n - 4, 4);
- } else if (n > 0) {
- dst[0] = char(src[0]);
- dst[n >> 1] = char(src[n >> 1]);
- dst[n - 1] = char(src[n - 1]);
- }
-}
-
-// Escapes the bytes of src in the range [i, blockend), given that m is the
-// (non-zero) escape mask for that range: the escape_mask_bits-wide lane of m
-// at byte k is non-zero when src[i + k] requires escaping. Returns the updated
-// output pointer.
-simdjson_never_inline char *escape_block(const uint8_t *src, char *out,
- size_t i, size_t blockend,
- uint64_t m) noexcept {
- constexpr uint64_t lane = (uint64_t(1) << escape_mask_bits) - 1;
-
- size_t pos = i; // first byte not yet copied
- while (m) {
- const size_t tz = trailing_zeroes(m);
- const size_t next = i + tz / escape_mask_bits;
- // Copy the run of safe bytes that precedes this escape.
- copy_lt16(out, src + pos, next - pos);
- out += next - pos;
- escape_json_char(char(src[next]), out);
- pos = next + 1;
- m &= ~(lane << tz);
- }
- // Copy whatever follows the last escape.
- copy_lt16(out, src + pos, blockend - pos);
- return out + (blockend - pos);
-}
-
-// Writes the escaped version of input to out, returning the number of bytes
-// written.
-simdjson_really_inline size_t write_string_escaped(const std::string_view input, char *out) {
- const size_t len = input.size();
- const uint8_t *src = reinterpret_cast<const uint8_t *>(input.data());
- const char *const initout = out;
-
- size_t i = 0;
-#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 && defined(__AVX2__)
- while (i + 32 <= len) {
- const __m256i word = _mm256_loadu_si256(reinterpret_cast<const __m256i *>(src + i));
- const __m256i flags = _mm256_or_si256(
- _mm256_or_si256(_mm256_cmpeq_epi8(word, _mm256_set1_epi8(34)), // '"'
- _mm256_cmpeq_epi8(word, _mm256_set1_epi8(92))), // '\\'
- _mm256_cmpeq_epi8(_mm256_subs_epu8(word, _mm256_set1_epi8(31)),
- _mm256_setzero_si256())); // control
- const uint32_t mask = uint32_t(_mm256_movemask_epi8(flags));
- if (simdjson_likely(mask == 0)) {
- _mm256_storeu_si256(reinterpret_cast<__m256i *>(out), word);
- out += 32;
- } else {
- for (size_t half = 0; half < 32; half += 16) {
- const uint64_t m = (mask >> half) & 0xFFFF;
- if (m == 0) {
- escape_store16(out, escape_load16(src + i + half));
- out += 16;
- } else {
- out = escape_block(src, out, i + half, i + half + 16, m);
- }
- }
- }
- i += 32;
- }
-#endif
- while (i + 16 <= len) {
- escape_vector word = escape_load16(src + i);
- escape_vector flags = escape_flags(word);
- if (simdjson_likely(!escape_any(flags))) {
- escape_store16(out, word);
- out += 16;
- } else {
- out = escape_block(src, out, i, i + 16, escape_bitmask(flags));
- }
- i += 16;
- }
- if (i < len) {
- const size_t rem = len - i;
- uint64_t m;
- if (len >= 16) {
- // The last 16 bytes of the input are in bounds. Bit k of that block's
- // mask belongs to input position len - 16 + k, so shift it down to align
- // bit 0 with position i.
- m = escape_mask(escape_load16(src + len - 16)) >>
- (escape_mask_bits * (16 - rem));
- } else if (len >= 8) {
- // Here i == 0 and rem == len. Two overlapping 8-byte loads cover
- // [0, 8) and [len - 8, len), which is the whole input since len < 16.
- uint64_t mm = escape_mask(escape_load8x2(src, src + len - 8));
- constexpr uint64_t low8 = (uint64_t(1) << (escape_mask_bits * 8)) - 1;
- m = (mm & low8) |
- ((mm >> (escape_mask_bits * 8)) << (escape_mask_bits * (len - 8)));
- } else if (len >= 4) {
- // Same idea with two overlapping 4-byte loads.
- uint64_t mm = escape_mask(escape_load4x2(src, src + len - 4));
- constexpr uint64_t low4 = (uint64_t(1) << (escape_mask_bits * 4)) - 1;
- m = (mm & low4) | (((mm >> (escape_mask_bits * 4)) & low4)
- << (escape_mask_bits * (len - 4)));
- } else {
- // Fewer than 4 bytes: at most three table lookups, no need for SIMD.
- for (size_t k = 0; k < len; k++) {
- uint8_t c = src[k];
- if (json_quotable_character[c]) {
- escape_json_char(char(c), out);
- } else {
- *out++ = char(c);
- }
- }
- return size_t(out - initout);
- }
- if (m == 0) {
- copy_lt16(out, src + i, rem);
- out += rem;
- } else {
- out = escape_block(src, out, i, len, m);
- }
- }
- return size_t(out - initout);
-}
-
-#else // SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-
-// Writes the escaped version of input to out, returning the number of bytes
-// written. Uses SIMD position finding to locate quotable characters efficiently.
-inline size_t write_string_escaped(const std::string_view input, char *out) {
- size_t mysize = input.size();
-
- // Use SIMD position finder directly - it returns mysize if no escape needed
- size_t location = find_next_json_quotable_character(input, 0);
- if (location == mysize) {
- // Fast path: no escaping needed
- memcpy(out, input.data(), input.size());
- return input.size();
- }
-
- const char *const initout = out;
- memcpy(out, input.data(), location);
- out += location;
- escape_json_char(input[location], out);
- location += 1;
- while (location < mysize) {
- size_t newlocation = find_next_json_quotable_character(input, location);
- memcpy(out, input.data() + location, newlocation - location);
- out += newlocation - location;
- location = newlocation;
- if (location == mysize) {
- break;
- }
- escape_json_char(input[location], out);
- location += 1;
- }
- return size_t(out - initout);
-}
-
-#endif // SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-#undef SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-
namespace internal {
// Sets the size of s to n without initializing the new bytes when the
@@ -71738,6 +67658,7 @@ simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t
/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */
/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_H */
/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */
+/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */
/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */
namespace simdjson {
@@ -71774,6 +67695,9 @@ struct has_custom_serialization : std::false_type {};
namespace rvv_vls {
namespace builder {
+
+using simdjson::internal::escape_best::write_string_escaped;
+
/**
* A builder for JSON strings representing documents. This is a low-level
* builder that is not meant to be used directly by end-users. Though it
@@ -72120,6 +68044,7 @@ simdjson_warn_unused error_code to_json(const Z &z, std::string &s, size_t initi
/* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */
/* amalgamation skipped (editor-only): #include "simdjson/concepts.h" */
/* amalgamation skipped (editor-only): #include "simdjson/annotations.h" */
+/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */
/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */
#if SIMDJSON_STATIC_REFLECTION
@@ -72145,7 +68070,6 @@ namespace internal {
using simdjson::internal::write_uint_jeaiii;
simdjson_inline char *write_double(char *p, double v) noexcept;
} // namespace internal
-simdjson_really_inline size_t write_string_escaped(const std::string_view input, char *out);
SIMDJSON_PUSH_DISABLE_WARNINGS
SIMDJSON_DISABLE_GCC_WARNING(-Warray-bounds)
@@ -73166,619 +69090,13 @@ simdjson_warn_unused simdjson_result<std::string> extract_fractured_json(
/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_INL_H */
/* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */
/* amalgamation skipped (editor-only): #include "simdjson/internal/jeaiii.h" */
+/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */
/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */
-/*
- * Empirically, we have found that an inlined optimization is important for
- * performance. The following macros are not ideal. We should find a better
- * way to inline the code.
- */
-
-#if defined(__SSE2__) || defined(__x86_64__) || defined(__x86_64) || \
- (defined(_M_AMD64) || defined(_M_X64) || \
- (defined(_M_IX86_FP) && _M_IX86_FP == 2))
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_SSE2
-#define SIMDJSON_EXPERIMENTAL_HAS_SSE2 1
-#endif
-#endif
-
-#if defined(__aarch64__) || defined(_M_ARM64)
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_NEON
-#define SIMDJSON_EXPERIMENTAL_HAS_NEON 1
-#endif
-#endif
-#if defined(__loongarch_sx)
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_LSX
-#define SIMDJSON_EXPERIMENTAL_HAS_LSX 1
-#endif
-#endif
-#if defined(__loongarch_asx)
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_LASX
-#define SIMDJSON_EXPERIMENTAL_HAS_LASX 1
-#endif
-#endif
-#if defined(__riscv_v_intrinsic) && __riscv_v_intrinsic >= 11000 && \
- defined(__riscv_vector)
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_RVV
-#define SIMDJSON_EXPERIMENTAL_HAS_RVV 1
-#endif
-#endif
-#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) && defined(__POWER8_VECTOR__)
-#ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64
-#define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1
-#endif
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_NEON
-#include <arm_neon.h>
-#ifdef _MSC_VER
-#include <intrin.h>
-#endif
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_SSE2
-#include <emmintrin.h>
-#if defined(__AVX2__)
-#include <immintrin.h>
-#endif
-#ifdef _MSC_VER
-#include <intrin.h>
-#endif
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_LSX
-#include <lsxintrin.h>
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_LASX
-#include <lasxintrin.h>
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_RVV
-#include <riscv_vector.h>
-#endif
-#if SIMDJSON_EXPERIMENTAL_HAS_PPC64
-#include <altivec.h>
-#ifdef bool
-#undef bool
-#endif
-#ifdef vector
-#undef vector
-#endif
-#endif
-
-
namespace simdjson {
namespace rvv_vls {
namespace builder {
-static SIMDJSON_CONSTEXPR_LAMBDA std::array<uint8_t, 256>
- json_quotable_character = {
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
-
-/**
-
-A possible SWAR implementation of has_json_escapable_byte. It is not used
-because it is slower than the current implementation. It is kept here for
-reference (to show that we tried it).
-
-inline bool has_json_escapable_byte(uint64_t x) {
- uint64_t is_ascii = 0x8080808080808080ULL & ~x;
- uint64_t xor2 = x ^ 0x0202020202020202ULL;
- uint64_t lt32_or_eq34 = xor2 - 0x2121212121212121ULL;
- uint64_t sub92 = x ^ 0x5C5C5C5C5C5C5C5CULL;
- uint64_t eq92 = (sub92 - 0x0101010101010101ULL);
- return ((lt32_or_eq34 | eq92) & is_ascii) != 0;
-}
-
-**/
-
-#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 || SIMDJSON_EXPERIMENTAL_HAS_NEON
-#define SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE 1
-#endif
-
-// With block escaping (SSE2/NEON), write_string_escaped scans the input itself.
-#if !SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-// Scalar fallback for finding next quotable character
-SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline size_t
-find_next_json_quotable_character_scalar(const std::string_view view,
- size_t location) noexcept {
- for (auto pos = view.begin() + location; pos != view.end(); ++pos) {
- if (json_quotable_character[static_cast<uint8_t>(*pos)]) {
- return pos - view.begin();
- }
- }
- return size_t(view.size());
-}
-
-// SIMD-accelerated position finding that directly locates the first quotable
-// character, combining detection and position extraction in a single pass to
-// minimize redundant work.
-#if SIMDJSON_EXPERIMENTAL_HAS_LASX
-simdjson_inline size_t
-find_next_json_quotable_character(const std::string_view view,
- size_t location) noexcept {
- const size_t len = view.size();
- const uint8_t *ptr =
- reinterpret_cast<const uint8_t *>(view.data()) + location;
- size_t remaining = len - location;
-
- // SIMD constants for characters requiring escape
- __m256i v34 = __lasx_xvreplgr2vr_b(34); // '"'
- __m256i v92 = __lasx_xvreplgr2vr_b(92); // '\\'
- __m256i v32 = __lasx_xvreplgr2vr_b(32); // control char threshold
-
- while (remaining >= 32) {
- __m256i word = __lasx_xvld(ptr, 0);
-
- // Check for quotable characters: '"', '\\', or control chars (< 32)
- __m256i needs_escape = __lasx_xvseq_b(word, v34);
- needs_escape = __lasx_xvor_v(needs_escape, __lasx_xvseq_b(word, v92));
- needs_escape = __lasx_xvor_v(needs_escape, __lasx_xvslt_bu(word, v32));
-
- if (!__lasx_xbz_v(needs_escape)) {
- // Found a quotable character - locate it via the four 64-bit lanes
- uint64_t lane0 = __lasx_xvpickve2gr_du(needs_escape, 0);
- uint64_t lane1 = __lasx_xvpickve2gr_du(needs_escape, 1);
- uint64_t lane2 = __lasx_xvpickve2gr_du(needs_escape, 2);
- uint64_t lane3 = __lasx_xvpickve2gr_du(needs_escape, 3);
- size_t offset = ptr - reinterpret_cast<const uint8_t *>(view.data());
- if (lane0 != 0) {
- return offset + trailing_zeroes(lane0) / 8;
- } else if (lane1 != 0) {
- return offset + 8 + trailing_zeroes(lane1) / 8;
- } else if (lane2 != 0) {
- return offset + 16 + trailing_zeroes(lane2) / 8;
- } else {
- return offset + 24 + trailing_zeroes(lane3) / 8;
- }
- }
- ptr += 32;
- remaining -= 32;
- }
- size_t current = len - remaining;
- return find_next_json_quotable_character_scalar(view, current);
-}
-#elif SIMDJSON_EXPERIMENTAL_HAS_LSX
-simdjson_inline size_t
-find_next_json_quotable_character(const std::string_view view,
- size_t location) noexcept {
- const size_t len = view.size();
- const uint8_t *ptr =
- reinterpret_cast<const uint8_t *>(view.data()) + location;
- size_t remaining = len - location;
-
- //SIMD constants for characters requiring escape
- __m128i v34 = __lsx_vreplgr2vr_b(34); // '"'
- __m128i v92 = __lsx_vreplgr2vr_b(92); // '\\'
- __m128i v32 = __lsx_vreplgr2vr_b(32); // control char threshold
-
- while (remaining >= 16){
- __m128i word = __lsx_vld(ptr, 0);
-
- //Check for the quotable characters: '"', '\\', or control char (<32)
- __m128i needs_escape = __lsx_vseq_b(word, v34);
- needs_escape = __lsx_vor_v(needs_escape, __lsx_vseq_b(word, v92));
- needs_escape = __lsx_vor_v(needs_escape, __lsx_vslt_bu(word, v32));
-
- if (!__lsx_bz_v(needs_escape)){
-
- //Found quotable character - extract exact byte position
- uint64_t lo = __lsx_vpickve2gr_du(needs_escape,0);
- uint64_t hi = __lsx_vpickve2gr_du(needs_escape,1);
- size_t offset = ptr - reinterpret_cast<const uint8_t *>(view.data());
- if ( lo != 0) {
- return offset + trailing_zeroes(lo) / 8;
- } else {
- return offset + 8 + trailing_zeroes(hi) / 8;
- }
- }
- ptr += 16;
- remaining -= 16;
- }
- size_t current = len - remaining;
- return find_next_json_quotable_character_scalar(view, current);
-}
-#elif SIMDJSON_EXPERIMENTAL_HAS_RVV
-simdjson_inline size_t
-find_next_json_quotable_character(const std::string_view view,
- size_t location) noexcept {
- const size_t len = view.size();
- const uint8_t *ptr =
- reinterpret_cast<const uint8_t *>(view.data()) + location;
- size_t remaining = len - location;
-
- while (remaining > 0) {
- size_t vl = __riscv_vsetvl_e8m1(remaining);
- vuint8m1_t word = __riscv_vle8_v_u8m1(ptr, vl);
-
- // Check for quotable characters: '"', '\\', or control chars (< 32)
- vbool8_t needs_escape = __riscv_vmseq(word, (uint8_t)34, vl);
- needs_escape = __riscv_vmor(needs_escape,
- __riscv_vmseq(word, (uint8_t)92, vl), vl);
- needs_escape = __riscv_vmor(needs_escape,
- __riscv_vmsltu(word, (uint8_t)32, vl), vl);
-
- long first = __riscv_vfirst(needs_escape, vl);
- if (first >= 0) {
- size_t offset = ptr - reinterpret_cast<const uint8_t *>(view.data());
- return offset + first;
- }
- ptr += vl;
- remaining -= vl;
- }
-
- return len;
-}
-#elif SIMDJSON_EXPERIMENTAL_HAS_PPC64
-simdjson_inline size_t
-find_next_json_quotable_character(const std::string_view view,
- size_t location) noexcept {
- const size_t len = view.size();
- const uint8_t *ptr =
- reinterpret_cast<const uint8_t *>(view.data()) + location;
- size_t remaining = len - location;
-
- // SIMD constants for characters requiring escape
- __vector unsigned char v34 = vec_splats((unsigned char)34); // '"'
- __vector unsigned char v92 = vec_splats((unsigned char)92); // '\\'
- __vector unsigned char v32 = vec_splats((unsigned char)32); // control char threshold
-
- // Bitmask for vec_vbpermq to extract one bit per byte
- const __vector unsigned char perm_mask = {0x78, 0x70, 0x68, 0x60, 0x58, 0x50,
- 0x48, 0x40, 0x38, 0x30, 0x28, 0x20,
- 0x18, 0x10, 0x08, 0x00};
-
- while (remaining >= 16) {
- __vector unsigned char word =
- vec_vsx_ld(0, reinterpret_cast<const unsigned char *>(ptr));
-
- // Check for quotable characters: '"', '\\', or control chars (< 32)
- __vector unsigned char needs_escape =
- (__vector unsigned char)vec_cmpeq(word, v34);
- needs_escape = vec_or(needs_escape,
- (__vector unsigned char)vec_cmpeq(word, v92));
- needs_escape = vec_or(needs_escape,
- (__vector unsigned char)vec_cmplt(word, v32));
-
- __vector unsigned long long result =
- (__vector unsigned long long)vec_vbpermq(needs_escape, perm_mask);
-#ifdef __LITTLE_ENDIAN__
- unsigned int mask = static_cast<unsigned int>(result[1]);
-#else
- unsigned int mask = static_cast<unsigned int>(result[0]);
-#endif
- if (mask != 0) {
- size_t offset = ptr - reinterpret_cast<const uint8_t *>(view.data());
- return offset + __builtin_ctz(mask);
- }
- ptr += 16;
- remaining -= 16;
- }
-
- // Scalar fallback for remaining bytes
- size_t current = len - remaining;
- return find_next_json_quotable_character_scalar(view, current);
-}
-#else
-SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline size_t
-find_next_json_quotable_character(const std::string_view view,
- size_t location) noexcept {
- return find_next_json_quotable_character_scalar(view, location);
-}
-#endif
-#endif // !SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-
-SIMDJSON_CONSTEXPR_LAMBDA static std::string_view control_chars[] = {
- "\\u0000", "\\u0001", "\\u0002", "\\u0003", "\\u0004", "\\u0005", "\\u0006",
- "\\u0007", "\\b", "\\t", "\\n", "\\u000b", "\\f", "\\r",
- "\\u000e", "\\u000f", "\\u0010", "\\u0011", "\\u0012", "\\u0013", "\\u0014",
- "\\u0015", "\\u0016", "\\u0017", "\\u0018", "\\u0019", "\\u001a", "\\u001b",
- "\\u001c", "\\u001d", "\\u001e", "\\u001f"};
-
-// All Unicode characters may be placed within the quotation marks, except for
-// the characters that MUST be escaped: quotation mark, reverse solidus, and the
-// control characters (U+0000 through U+001F). There are two-character sequence
-// escape representations of some popular characters:
-// \", \\, \b, \f, \n, \r, \t.
-SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline void escape_json_char(char c, char *&out) {
- if (c == '"') {
- memcpy(out, "\\\"", 2);
- out += 2;
- } else if (c == '\\') {
- memcpy(out, "\\\\", 2);
- out += 2;
- } else {
- std::string_view v = control_chars[uint8_t(c)];
- memcpy(out, v.data(), v.size());
- out += v.size();
- }
-}
-
-#if SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-
-#if SIMDJSON_EXPERIMENTAL_HAS_SSE2
-
-using escape_vector = __m128i;
-// Mask bits that each input byte contributes to escape_bitmask().
-static constexpr unsigned escape_mask_bits = 1;
-
-simdjson_inline escape_vector escape_load16(const uint8_t *p) noexcept {
- return _mm_loadu_si128(reinterpret_cast<const __m128i *>(p));
-}
-
-simdjson_inline void escape_store16(char *out, escape_vector v) noexcept {
- _mm_storeu_si128(reinterpret_cast<__m128i *>(out), v);
-}
-
-// Builds a vector whose bytes 0..7 come from a and bytes 8..15 from b.
-simdjson_inline escape_vector escape_load8x2(const uint8_t *a,
- const uint8_t *b) noexcept {
- return _mm_unpacklo_epi64(
- _mm_loadl_epi64(reinterpret_cast<const __m128i *>(a)),
- _mm_loadl_epi64(reinterpret_cast<const __m128i *>(b)));
-}
-
-// Builds a vector whose bytes 0..3 come from a and bytes 4..7 from b. The
-// upper half is unspecified; callers only look at the low eight bits.
-simdjson_inline escape_vector escape_load4x2(const uint8_t *a,
- const uint8_t *b) noexcept {
- int32_t a32, b32;
- memcpy(&a32, a, 4);
- memcpy(&b32, b, 4);
- return _mm_unpacklo_epi32(_mm_cvtsi32_si128(a32), _mm_cvtsi32_si128(b32));
-}
-
-// Sets every bit of byte k when byte k of v is a quotable character ('"',
-// '\\' or a control character).
-simdjson_inline escape_vector escape_flags(escape_vector v) noexcept {
- const __m128i v34 = _mm_set1_epi8(34); // '"'
- const __m128i v92 = _mm_set1_epi8(92); // '\\'
- const __m128i v31 = _mm_set1_epi8(31); // for control char detection
- __m128i needs_escape = _mm_cmpeq_epi8(v, v34);
- needs_escape = _mm_or_si128(needs_escape, _mm_cmpeq_epi8(v, v92));
- return _mm_or_si128(
- needs_escape, _mm_cmpeq_epi8(_mm_subs_epu8(v, v31), _mm_setzero_si128()));
-}
-
-// True when any byte needs escaping. Kept separate from escape_bitmask
-// because some instruction sets can answer it without leaving the vector
-// register file; on SSE2 the compiler folds the two together.
-simdjson_inline bool escape_any(escape_vector flags) noexcept {
- return _mm_movemask_epi8(flags) != 0;
-}
-
-// A 16-bit mask with bit k set when byte k needed escaping.
-simdjson_inline uint64_t escape_bitmask(escape_vector flags) noexcept {
- return uint64_t(uint32_t(_mm_movemask_epi8(flags)));
-}
-
-#else // SIMDJSON_EXPERIMENTAL_HAS_NEON
-
-using escape_vector = uint8x16_t;
-static constexpr unsigned escape_mask_bits = 4;
-
-simdjson_inline escape_vector escape_load16(const uint8_t *p) noexcept {
- return vld1q_u8(p);
-}
-
-simdjson_inline void escape_store16(char *out, escape_vector v) noexcept {
- vst1q_u8(reinterpret_cast<uint8_t *>(out), v);
-}
-
-simdjson_inline escape_vector escape_load8x2(const uint8_t *a,
- const uint8_t *b) noexcept {
- uint64_t a64, b64;
- memcpy(&a64, a, 8);
- memcpy(&b64, b, 8);
- return vreinterpretq_u8_u64(vsetq_lane_u64(b64, vdupq_n_u64(a64), 1));
-}
-
-simdjson_inline escape_vector escape_load4x2(const uint8_t *a,
- const uint8_t *b) noexcept {
- uint32_t a32, b32;
- memcpy(&a32, a, 4);
- memcpy(&b32, b, 4);
- return vreinterpretq_u8_u32(vsetq_lane_u32(b32, vdupq_n_u32(a32), 1));
-}
-
-simdjson_inline escape_vector escape_flags(escape_vector v) noexcept {
- uint8x16_t needs_escape = vceqq_u8(v, vdupq_n_u8(34)); // '"'
- needs_escape = vorrq_u8(needs_escape, vceqq_u8(v, vdupq_n_u8(92))); // '\\'
- return vorrq_u8(needs_escape, vcltq_u8(v, vdupq_n_u8(32)));
-}
-
-simdjson_inline bool escape_any(escape_vector flags) noexcept {
- return vmaxvq_u32(vreinterpretq_u32_u8(flags)) != 0;
-}
-
-// Four bits per byte rather than one: escape_block and the tail paths scale
-// their shifts by escape_mask_bits to match.
-simdjson_inline uint64_t escape_bitmask(escape_vector flags) noexcept {
- uint8x8_t narrowed = vshrn_n_u16(vreinterpretq_u16_u8(flags), 4);
- return vget_lane_u64(vreinterpret_u64_u8(narrowed), 0);
-}
-
-#endif // instruction set selection
-
-// The escape bitmask of a 16-byte block.
-simdjson_inline uint64_t escape_mask(escape_vector v) noexcept {
- return escape_bitmask(escape_flags(v));
-}
-
-// Copies n bytes with n < 16, using overlapping loads and stores. It never
-// reads more than n bytes from src, nor writes more than n bytes to dst.
-simdjson_inline void copy_lt16(char *dst, const uint8_t *src,
- size_t n) noexcept {
- if (n >= 8) {
- memcpy(dst, src, 8);
- memcpy(dst + n - 8, src + n - 8, 8);
- } else if (n >= 4) {
- memcpy(dst, src, 4);
- memcpy(dst + n - 4, src + n - 4, 4);
- } else if (n > 0) {
- dst[0] = char(src[0]);
- dst[n >> 1] = char(src[n >> 1]);
- dst[n - 1] = char(src[n - 1]);
- }
-}
-
-// Escapes the bytes of src in the range [i, blockend), given that m is the
-// (non-zero) escape mask for that range: the escape_mask_bits-wide lane of m
-// at byte k is non-zero when src[i + k] requires escaping. Returns the updated
-// output pointer.
-simdjson_never_inline char *escape_block(const uint8_t *src, char *out,
- size_t i, size_t blockend,
- uint64_t m) noexcept {
- constexpr uint64_t lane = (uint64_t(1) << escape_mask_bits) - 1;
-
- size_t pos = i; // first byte not yet copied
- while (m) {
- const size_t tz = trailing_zeroes(m);
- const size_t next = i + tz / escape_mask_bits;
- // Copy the run of safe bytes that precedes this escape.
- copy_lt16(out, src + pos, next - pos);
- out += next - pos;
- escape_json_char(char(src[next]), out);
- pos = next + 1;
- m &= ~(lane << tz);
- }
- // Copy whatever follows the last escape.
- copy_lt16(out, src + pos, blockend - pos);
- return out + (blockend - pos);
-}
-
-// Writes the escaped version of input to out, returning the number of bytes
-// written.
-simdjson_really_inline size_t write_string_escaped(const std::string_view input, char *out) {
- const size_t len = input.size();
- const uint8_t *src = reinterpret_cast<const uint8_t *>(input.data());
- const char *const initout = out;
-
- size_t i = 0;
-#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 && defined(__AVX2__)
- while (i + 32 <= len) {
- const __m256i word = _mm256_loadu_si256(reinterpret_cast<const __m256i *>(src + i));
- const __m256i flags = _mm256_or_si256(
- _mm256_or_si256(_mm256_cmpeq_epi8(word, _mm256_set1_epi8(34)), // '"'
- _mm256_cmpeq_epi8(word, _mm256_set1_epi8(92))), // '\\'
- _mm256_cmpeq_epi8(_mm256_subs_epu8(word, _mm256_set1_epi8(31)),
- _mm256_setzero_si256())); // control
- const uint32_t mask = uint32_t(_mm256_movemask_epi8(flags));
- if (simdjson_likely(mask == 0)) {
- _mm256_storeu_si256(reinterpret_cast<__m256i *>(out), word);
- out += 32;
- } else {
- for (size_t half = 0; half < 32; half += 16) {
- const uint64_t m = (mask >> half) & 0xFFFF;
- if (m == 0) {
- escape_store16(out, escape_load16(src + i + half));
- out += 16;
- } else {
- out = escape_block(src, out, i + half, i + half + 16, m);
- }
- }
- }
- i += 32;
- }
-#endif
- while (i + 16 <= len) {
- escape_vector word = escape_load16(src + i);
- escape_vector flags = escape_flags(word);
- if (simdjson_likely(!escape_any(flags))) {
- escape_store16(out, word);
- out += 16;
- } else {
- out = escape_block(src, out, i, i + 16, escape_bitmask(flags));
- }
- i += 16;
- }
- if (i < len) {
- const size_t rem = len - i;
- uint64_t m;
- if (len >= 16) {
- // The last 16 bytes of the input are in bounds. Bit k of that block's
- // mask belongs to input position len - 16 + k, so shift it down to align
- // bit 0 with position i.
- m = escape_mask(escape_load16(src + len - 16)) >>
- (escape_mask_bits * (16 - rem));
- } else if (len >= 8) {
- // Here i == 0 and rem == len. Two overlapping 8-byte loads cover
- // [0, 8) and [len - 8, len), which is the whole input since len < 16.
- uint64_t mm = escape_mask(escape_load8x2(src, src + len - 8));
- constexpr uint64_t low8 = (uint64_t(1) << (escape_mask_bits * 8)) - 1;
- m = (mm & low8) |
- ((mm >> (escape_mask_bits * 8)) << (escape_mask_bits * (len - 8)));
- } else if (len >= 4) {
- // Same idea with two overlapping 4-byte loads.
- uint64_t mm = escape_mask(escape_load4x2(src, src + len - 4));
- constexpr uint64_t low4 = (uint64_t(1) << (escape_mask_bits * 4)) - 1;
- m = (mm & low4) | (((mm >> (escape_mask_bits * 4)) & low4)
- << (escape_mask_bits * (len - 4)));
- } else {
- // Fewer than 4 bytes: at most three table lookups, no need for SIMD.
- for (size_t k = 0; k < len; k++) {
- uint8_t c = src[k];
- if (json_quotable_character[c]) {
- escape_json_char(char(c), out);
- } else {
- *out++ = char(c);
- }
- }
- return size_t(out - initout);
- }
- if (m == 0) {
- copy_lt16(out, src + i, rem);
- out += rem;
- } else {
- out = escape_block(src, out, i, len, m);
- }
- }
- return size_t(out - initout);
-}
-
-#else // SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-
-// Writes the escaped version of input to out, returning the number of bytes
-// written. Uses SIMD position finding to locate quotable characters efficiently.
-inline size_t write_string_escaped(const std::string_view input, char *out) {
- size_t mysize = input.size();
-
- // Use SIMD position finder directly - it returns mysize if no escape needed
- size_t location = find_next_json_quotable_character(input, 0);
- if (location == mysize) {
- // Fast path: no escaping needed
- memcpy(out, input.data(), input.size());
- return input.size();
- }
-
- const char *const initout = out;
- memcpy(out, input.data(), location);
- out += location;
- escape_json_char(input[location], out);
- location += 1;
- while (location < mysize) {
- size_t newlocation = find_next_json_quotable_character(input, location);
- memcpy(out, input.data() + location, newlocation - location);
- out += newlocation - location;
- location = newlocation;
- if (location == mysize) {
- break;
- }
- escape_json_char(input[location], out);
- location += 1;
- }
- return size_t(out - initout);
-}
-
-#endif // SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-#undef SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE
-
namespace internal {
// Sets the size of s to n without initializing the new bytes when the
@@ -76022,8 +71340,13 @@ public:
* Create a new invalid value.
*
* Exists so you can declare a variable and later assign to it before use.
+ *
+ * It is user-provided (not = default) on purpose: value-initialization (std::pair, T{})
+ * then skips zeroing the whole object, which led GCC 15 and 16 to keep
+ * simdjson_result<field> on the stack when iterating over objects, causing
+ * store-forwarding stalls (up to 2x slower).
*/
- simdjson_inline value() noexcept = default;
+ simdjson_inline value() noexcept {}
/**
* Get this value as the given type.
@@ -87306,7 +82629,7 @@ simdjson_inline document_stream::iterator document_stream::end() noexcept {
inline void document_stream::start() noexcept {
if (error) { return; }
- error = parser->allocate(batch_size);
+ error = parser->allocate(batch_size, parser->max_depth());
if (error) { return; }
// Always run the first stage 1 parse immediately
batch_start = 0;
@@ -87332,7 +82655,10 @@ inline void document_stream::start() noexcept {
worker.reset(new(std::nothrow) stage1_worker());
if (worker.get() == nullptr) { error = MEMALLOC; return; }
}
- error = stage1_thread_parser.allocate(batch_size);
+ // load_from_stage1_thread() swaps *parser with stage1_thread_parser, which
+ // then parses every other batch: give it the same settings.
+ stage1_thread_parser.set_max_capacity(parser->max_capacity());
+ error = stage1_thread_parser.allocate(batch_size, parser->max_depth());
if (error) { return; }
worker->start_thread();
start_stage1_thread();
@@ -93507,8 +88833,13 @@ public:
* Create a new invalid value.
*
* Exists so you can declare a variable and later assign to it before use.
+ *
+ * It is user-provided (not = default) on purpose: value-initialization (std::pair, T{})
+ * then skips zeroing the whole object, which led GCC 15 and 16 to keep
+ * simdjson_result<field> on the stack when iterating over objects, causing
+ * store-forwarding stalls (up to 2x slower).
*/
- simdjson_inline value() noexcept = default;
+ simdjson_inline value() noexcept {}
/**
* Get this value as the given type.
@@ -104791,7 +100122,7 @@ simdjson_inline document_stream::iterator document_stream::end() noexcept {
inline void document_stream::start() noexcept {
if (error) { return; }
- error = parser->allocate(batch_size);
+ error = parser->allocate(batch_size, parser->max_depth());
if (error) { return; }
// Always run the first stage 1 parse immediately
batch_start = 0;
@@ -104817,7 +100148,10 @@ inline void document_stream::start() noexcept {
worker.reset(new(std::nothrow) stage1_worker());
if (worker.get() == nullptr) { error = MEMALLOC; return; }
}
- error = stage1_thread_parser.allocate(batch_size);
+ // load_from_stage1_thread() swaps *parser with stage1_thread_parser, which
+ // then parses every other batch: give it the same settings.
+ stage1_thread_parser.set_max_capacity(parser->max_capacity());
+ error = stage1_thread_parser.allocate(batch_size, parser->max_depth());
if (error) { return; }
worker->start_thread();
start_stage1_thread();
@@ -111462,8 +106796,13 @@ public:
* Create a new invalid value.
*
* Exists so you can declare a variable and later assign to it before use.
+ *
+ * It is user-provided (not = default) on purpose: value-initialization (std::pair, T{})
+ * then skips zeroing the whole object, which led GCC 15 and 16 to keep
+ * simdjson_result<field> on the stack when iterating over objects, causing
+ * store-forwarding stalls (up to 2x slower).
*/
- simdjson_inline value() noexcept = default;
+ simdjson_inline value() noexcept {}
/**
* Get this value as the given type.
@@ -122746,7 +118085,7 @@ simdjson_inline document_stream::iterator document_stream::end() noexcept {
inline void document_stream::start() noexcept {
if (error) { return; }
- error = parser->allocate(batch_size);
+ error = parser->allocate(batch_size, parser->max_depth());
if (error) { return; }
// Always run the first stage 1 parse immediately
batch_start = 0;
@@ -122772,7 +118111,10 @@ inline void document_stream::start() noexcept {
worker.reset(new(std::nothrow) stage1_worker());
if (worker.get() == nullptr) { error = MEMALLOC; return; }
}
- error = stage1_thread_parser.allocate(batch_size);
+ // load_from_stage1_thread() swaps *parser with stage1_thread_parser, which
+ // then parses every other batch: give it the same settings.
+ stage1_thread_parser.set_max_capacity(parser->max_capacity());
+ error = stage1_thread_parser.allocate(batch_size, parser->max_depth());
if (error) { return; }
worker->start_thread();
start_stage1_thread();
@@ -129413,8 +124755,13 @@ public:
* Create a new invalid value.
*
* Exists so you can declare a variable and later assign to it before use.
+ *
+ * It is user-provided (not = default) on purpose: value-initialization (std::pair, T{})
+ * then skips zeroing the whole object, which led GCC 15 and 16 to keep
+ * simdjson_result<field> on the stack when iterating over objects, causing
+ * store-forwarding stalls (up to 2x slower).
*/
- simdjson_inline value() noexcept = default;
+ simdjson_inline value() noexcept {}
/**
* Get this value as the given type.
@@ -140697,7 +136044,7 @@ simdjson_inline document_stream::iterator document_stream::end() noexcept {
inline void document_stream::start() noexcept {
if (error) { return; }
- error = parser->allocate(batch_size);
+ error = parser->allocate(batch_size, parser->max_depth());
if (error) { return; }
// Always run the first stage 1 parse immediately
batch_start = 0;
@@ -140723,7 +136070,10 @@ inline void document_stream::start() noexcept {
worker.reset(new(std::nothrow) stage1_worker());
if (worker.get() == nullptr) { error = MEMALLOC; return; }
}
- error = stage1_thread_parser.allocate(batch_size);
+ // load_from_stage1_thread() swaps *parser with stage1_thread_parser, which
+ // then parses every other batch: give it the same settings.
+ stage1_thread_parser.set_max_capacity(parser->max_capacity());
+ error = stage1_thread_parser.allocate(batch_size, parser->max_depth());
if (error) { return; }
worker->start_thread();
start_stage1_thread();
@@ -147482,8 +142832,13 @@ public:
* Create a new invalid value.
*
* Exists so you can declare a variable and later assign to it before use.
+ *
+ * It is user-provided (not = default) on purpose: value-initialization (std::pair, T{})
+ * then skips zeroing the whole object, which led GCC 15 and 16 to keep
+ * simdjson_result<field> on the stack when iterating over objects, causing
+ * store-forwarding stalls (up to 2x slower).
*/
- simdjson_inline value() noexcept = default;
+ simdjson_inline value() noexcept {}
/**
* Get this value as the given type.
@@ -158766,7 +154121,7 @@ simdjson_inline document_stream::iterator document_stream::end() noexcept {
inline void document_stream::start() noexcept {
if (error) { return; }
- error = parser->allocate(batch_size);
+ error = parser->allocate(batch_size, parser->max_depth());
if (error) { return; }
// Always run the first stage 1 parse immediately
batch_start = 0;
@@ -158792,7 +154147,10 @@ inline void document_stream::start() noexcept {
worker.reset(new(std::nothrow) stage1_worker());
if (worker.get() == nullptr) { error = MEMALLOC; return; }
}
- error = stage1_thread_parser.allocate(batch_size);
+ // load_from_stage1_thread() swaps *parser with stage1_thread_parser, which
+ // then parses every other batch: give it the same settings.
+ stage1_thread_parser.set_max_capacity(parser->max_capacity());
+ error = stage1_thread_parser.allocate(batch_size, parser->max_depth());
if (error) { return; }
worker->start_thread();
start_stage1_thread();
@@ -165859,8 +161217,13 @@ public:
* Create a new invalid value.
*
* Exists so you can declare a variable and later assign to it before use.
+ *
+ * It is user-provided (not = default) on purpose: value-initialization (std::pair, T{})
+ * then skips zeroing the whole object, which led GCC 15 and 16 to keep
+ * simdjson_result<field> on the stack when iterating over objects, causing
+ * store-forwarding stalls (up to 2x slower).
*/
- simdjson_inline value() noexcept = default;
+ simdjson_inline value() noexcept {}
/**
* Get this value as the given type.
@@ -177143,7 +172506,7 @@ simdjson_inline document_stream::iterator document_stream::end() noexcept {
inline void document_stream::start() noexcept {
if (error) { return; }
- error = parser->allocate(batch_size);
+ error = parser->allocate(batch_size, parser->max_depth());
if (error) { return; }
// Always run the first stage 1 parse immediately
batch_start = 0;
@@ -177169,7 +172532,10 @@ inline void document_stream::start() noexcept {
worker.reset(new(std::nothrow) stage1_worker());
if (worker.get() == nullptr) { error = MEMALLOC; return; }
}
- error = stage1_thread_parser.allocate(batch_size);
+ // load_from_stage1_thread() swaps *parser with stage1_thread_parser, which
+ // then parses every other batch: give it the same settings.
+ stage1_thread_parser.set_max_capacity(parser->max_capacity());
+ error = stage1_thread_parser.allocate(batch_size, parser->max_depth());
if (error) { return; }
worker->start_thread();
start_stage1_thread();
@@ -183726,8 +179092,13 @@ public:
* Create a new invalid value.
*
* Exists so you can declare a variable and later assign to it before use.
+ *
+ * It is user-provided (not = default) on purpose: value-initialization (std::pair, T{})
+ * then skips zeroing the whole object, which led GCC 15 and 16 to keep
+ * simdjson_result<field> on the stack when iterating over objects, causing
+ * store-forwarding stalls (up to 2x slower).
*/
- simdjson_inline value() noexcept = default;
+ simdjson_inline value() noexcept {}
/**
* Get this value as the given type.
@@ -195010,7 +190381,7 @@ simdjson_inline document_stream::iterator document_stream::end() noexcept {
inline void document_stream::start() noexcept {
if (error) { return; }
- error = parser->allocate(batch_size);
+ error = parser->allocate(batch_size, parser->max_depth());
if (error) { return; }
// Always run the first stage 1 parse immediately
batch_start = 0;
@@ -195036,7 +190407,10 @@ inline void document_stream::start() noexcept {
worker.reset(new(std::nothrow) stage1_worker());
if (worker.get() == nullptr) { error = MEMALLOC; return; }
}
- error = stage1_thread_parser.allocate(batch_size);
+ // load_from_stage1_thread() swaps *parser with stage1_thread_parser, which
+ // then parses every other batch: give it the same settings.
+ stage1_thread_parser.set_max_capacity(parser->max_capacity());
+ error = stage1_thread_parser.allocate(batch_size, parser->max_depth());
if (error) { return; }
worker->start_thread();
start_stage1_thread();
@@ -201616,8 +196990,13 @@ public:
* Create a new invalid value.
*
* Exists so you can declare a variable and later assign to it before use.
+ *
+ * It is user-provided (not = default) on purpose: value-initialization (std::pair, T{})
+ * then skips zeroing the whole object, which led GCC 15 and 16 to keep
+ * simdjson_result<field> on the stack when iterating over objects, causing
+ * store-forwarding stalls (up to 2x slower).
*/
- simdjson_inline value() noexcept = default;
+ simdjson_inline value() noexcept {}
/**
* Get this value as the given type.
@@ -212900,7 +208279,7 @@ simdjson_inline document_stream::iterator document_stream::end() noexcept {
inline void document_stream::start() noexcept {
if (error) { return; }
- error = parser->allocate(batch_size);
+ error = parser->allocate(batch_size, parser->max_depth());
if (error) { return; }
// Always run the first stage 1 parse immediately
batch_start = 0;
@@ -212926,7 +208305,10 @@ inline void document_stream::start() noexcept {
worker.reset(new(std::nothrow) stage1_worker());
if (worker.get() == nullptr) { error = MEMALLOC; return; }
}
- error = stage1_thread_parser.allocate(batch_size);
+ // load_from_stage1_thread() swaps *parser with stage1_thread_parser, which
+ // then parses every other batch: give it the same settings.
+ stage1_thread_parser.set_max_capacity(parser->max_capacity());
+ error = stage1_thread_parser.allocate(batch_size, parser->max_depth());
if (error) { return; }
worker->start_thread();
start_stage1_thread();
@@ -219516,8 +214898,13 @@ public:
* Create a new invalid value.
*
* Exists so you can declare a variable and later assign to it before use.
+ *
+ * It is user-provided (not = default) on purpose: value-initialization (std::pair, T{})
+ * then skips zeroing the whole object, which led GCC 15 and 16 to keep
+ * simdjson_result<field> on the stack when iterating over objects, causing
+ * store-forwarding stalls (up to 2x slower).
*/
- simdjson_inline value() noexcept = default;
+ simdjson_inline value() noexcept {}
/**
* Get this value as the given type.
@@ -230800,7 +226187,7 @@ simdjson_inline document_stream::iterator document_stream::end() noexcept {
inline void document_stream::start() noexcept {
if (error) { return; }
- error = parser->allocate(batch_size);
+ error = parser->allocate(batch_size, parser->max_depth());
if (error) { return; }
// Always run the first stage 1 parse immediately
batch_start = 0;
@@ -230826,7 +226213,10 @@ inline void document_stream::start() noexcept {
worker.reset(new(std::nothrow) stage1_worker());
if (worker.get() == nullptr) { error = MEMALLOC; return; }
}
- error = stage1_thread_parser.allocate(batch_size);
+ // load_from_stage1_thread() swaps *parser with stage1_thread_parser, which
+ // then parses every other batch: give it the same settings.
+ stage1_thread_parser.set_max_capacity(parser->max_capacity());
+ error = stage1_thread_parser.allocate(batch_size, parser->max_depth());
if (error) { return; }
worker->start_thread();
start_stage1_thread();