Commit 6d9c82ea2 for llama.cpp
commit 6d9c82ea2bb34e277c0664b8dd3434bfb4dcfb27
Author: Todor Boinovski <todorb@qti.qualcomm.com>
Date: Wed Sep 9 08:40:24 2026 -0700
hexagon: rope updates (#28628)
* hexagon: vectorize RoPE theta cache on v75
* hexagon: vectorize MROPE/IMROPE theta pick
* hexagon: tighten NEOX RoPE rotate and aligned tail copy
* hex-rope: use inplace rope for all scenarios
* hex-rope: remove ctx->spad usage and legacy timers
* hex-rope: add kernel params and enforce vtcm reqs at the host
* hex-rope: cleanup unused params and tighten the mode checks
* hex-rope: add missing ops header
---------
Co-authored-by: Max Krasnyansky <maxk@qti.qualcomm.com>
diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp
index a39df2a87..112e9bae6 100644
--- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp
+++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp
@@ -56,6 +56,7 @@
#include "htp/unary-ops.h"
#include "htp/get-rows-ops.h"
#include "htp/set-rows-ops.h"
+#include "htp/rope-ops.h"
#include "htp_iface.h"
#include "htp-drv.h"
@@ -299,6 +300,12 @@ static void ggml_hexagon_precompute_set_rows_params(
struct htp_set_rows_kernel_params * kparams
);
+static void ggml_hexagon_precompute_rope_params(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * op,
+ struct htp_rope_kernel_params * kparams
+);
+
static void ggml_hexagon_precompute_fused_mmnx_params(
const struct ggml_hexagon_session * sess,
const struct ggml_tensor * src0,
@@ -4148,6 +4155,36 @@ static void ggml_hexagon_precompute_set_rows_params(
kparams->vtcm_size = vtcm_layout.total_bytes;
}
+static void ggml_hexagon_precompute_rope_params(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * op,
+ struct htp_rope_kernel_params * kparams
+) {
+ memset(kparams, 0, sizeof(*kparams));
+
+ const struct ggml_tensor * src0 = op->src[0];
+ const struct ggml_tensor * dst = op;
+
+ const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
+ const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, src0_nrows);
+
+ struct htp_rope_vtcm_layout layout;
+ htp_rope_vtcm_layout_build(&layout, src0->ne[0], n_threads);
+
+ kparams->n_threads = n_threads;
+ kparams->src0_nrows = src0_nrows;
+ kparams->src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads;
+ kparams->vtcm_size = (uint32_t) layout.total_bytes;
+ kparams->spad_per_thread = (uint32_t) layout.bytes_per_thread;
+ kparams->theta_cache_offset = (uint32_t) layout.theta_cache_size_aligned;
+ kparams->src0_row_size_aligned = (uint32_t) layout.src0_row_size_aligned;
+
+ if (src0_nrows > 0) {
+ kparams->div_ne2_ne1 = init_fastdiv_values(dst->ne[2] * dst->ne[1]);
+ kparams->div_ne1 = init_fastdiv_values(dst->ne[1]);
+ }
+}
+
static void ggml_hexagon_precompute_fused_mmnx_params(
const struct ggml_hexagon_session * sess,
const struct ggml_tensor * src0, // W0
@@ -4706,56 +4743,82 @@ static bool ggml_hexagon_supported_argsort(const struct ggml_hexagon_session * s
}
static bool ggml_hexagon_supported_rope(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
- const int32_t * op_params = &op->op_params[0];
+ const struct ggml_tensor * src0 = op->src[0];
+ const struct ggml_tensor * src1 = op->src[1];
+ const struct ggml_tensor * src2 = op->src[2];
+ const struct ggml_tensor * dst = op;
- // ggml_rope_set_offset: HVX kernels need a VLEN-aligned window start (32 f32 elems)
- if (op_params[15] % 32 != 0) {
+ if (!ggml_are_same_shape(src0, dst)) {
return false;
}
- int mode = op_params[2];
+ if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_I32) {
+ return false;
+ }
- // n_dims == ne0/2, so the rotation spans the full row
- if (mode == GGML_ROPE_TYPE_VISION) {
- const int n_dims = op_params[1];
- if (n_dims != (int) (op->src[0]->ne[0] / 2)) {
- return false;
- }
+ if (src0->ne[0] <= 0) {
+ return false;
}
- if (mode & 1) {
+
+ const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
+ if (src0_nrows == 0) {
return false;
}
- const struct ggml_tensor * src0 = op->src[0];
- const struct ggml_tensor * src1 = op->src[1];
- const struct ggml_tensor * src2 = op->src[2];
- const struct ggml_tensor * dst = op;
+ const int32_t * op_params = &op->op_params[0];
+ const int n_dims = op_params[1];
+ const int mode = op_params[2];
+ const int n_offs = op_params[15];
- if (src0->type != GGML_TYPE_F32) {
- return false; // FIXME: add support for GGML_TYPE_F16 for src0
+ if (n_dims <= 0 || n_dims % 2 != 0) {
+ return false;
}
- if (dst->type != GGML_TYPE_F32) {
+
+ // ggml_rope_set_offset: HVX kernels need a VLEN-aligned window start (32 f32 elems)
+ if (n_offs < 0 || (n_offs % 32 != 0) || (n_offs + n_dims > src0->ne[0])) {
return false;
}
- if (src1->type != GGML_TYPE_I32) {
+
+ float freq_base;
+ memcpy(&freq_base, op_params + 5, sizeof(float));
+ if (freq_base <= 0.0f) {
return false;
}
- if (src2) {
- if (src2->type != GGML_TYPE_F32) {
+
+ if (mode != GGML_ROPE_TYPE_NORMAL &&
+ mode != GGML_ROPE_TYPE_NEOX &&
+ mode != GGML_ROPE_TYPE_MROPE &&
+ mode != GGML_ROPE_TYPE_VISION &&
+ mode != GGML_ROPE_TYPE_IMROPE) {
+ return false;
+ }
+
+ const bool is_mrope = (mode & GGML_ROPE_TYPE_MROPE) != 0;
+
+ // n_dims == ne0/2, so the rotation spans the full row
+ if (mode == GGML_ROPE_TYPE_VISION) {
+ if (n_dims != (int) (src0->ne[0] / 2) || n_offs != 0) {
return false;
}
- int n_dims = op_params[1];
- if (src2->ne[0] < (n_dims / 2)) {
+ }
+
+ if (is_mrope) {
+ const int32_t * sections = op_params + 11;
+ if (sections[0] <= 0 && sections[1] <= 0 && sections[2] <= 0) {
return false;
}
}
+ const int64_t min_pos_len = (is_mrope || mode == GGML_ROPE_TYPE_VISION) ? src0->ne[2] * 4 : src0->ne[2];
+ if (src1->ne[0] < min_pos_len || !ggml_is_contiguous(src1)) {
+ return false;
+ }
+
if (src2) {
- if (!ggml_is_contiguous(src1) || !ggml_is_contiguous(src2)) {
+ if (src2->type != GGML_TYPE_F32 || !ggml_is_contiguous(src2)) {
return false;
}
- } else {
- if (!ggml_is_contiguous(src1)) {
+ if (src2->ne[0] < (n_dims / 2)) {
return false;
}
}
@@ -4768,9 +4831,16 @@ static bool ggml_hexagon_supported_rope(const struct ggml_hexagon_session * sess
if (src0->nb[1] < src0->ne[0] * sizeof(float) || dst->nb[1] < dst->ne[0] * sizeof(float)) {
return false;
}
- return true;
- GGML_UNUSED(sess);
+ const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, src0_nrows);
+
+ struct htp_rope_vtcm_layout layout;
+ htp_rope_vtcm_layout_build(&layout, src0->ne[0], n_threads);
+ if (layout.total_bytes > sess->vtcm_size) {
+ return false;
+ }
+
+ return true;
}
static bool ggml_hexagon_supported_ssm_conv(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
@@ -5206,6 +5276,11 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, gg
node.node->src[0], node.node->src[1], node.dst(),
(struct htp_set_rows_kernel_params *)node.kernel_params
);
+ } else if (node.opcode == HTP_OP_ROPE) {
+ ggml_hexagon_precompute_rope_params(sess,
+ node.node,
+ (struct htp_rope_kernel_params *)node.kernel_params
+ );
}
computed_nodes.push_back(std::move(node));
}
diff --git a/ggml/src/ggml-hexagon/htp/hvx-sin-cos.h b/ggml/src/ggml-hexagon/htp/hvx-sin-cos.h
index c5b9a5d47..8648af0e5 100644
--- a/ggml/src/ggml-hexagon/htp/hvx-sin-cos.h
+++ b/ggml/src/ggml-hexagon/htp/hvx-sin-cos.h
@@ -4,87 +4,75 @@
#include "hvx-base.h"
#include "hvx-floor.h"
-static inline HVX_Vector hvx_vec_cos_f32(HVX_Vector x) {
- HVX_Vector const_inv_pi = hvx_vec_splat_f32(0.3183098861837907f);
- HVX_Vector const_half = hvx_vec_splat_f32(0.5f);
- HVX_Vector const_pi = hvx_vec_splat_f32(3.141592653589793f);
- HVX_Vector const_one = hvx_vec_splat_f32(1.0f);
+// Range-reduce x to y in [-pi/2, pi/2] and the quadrant sign (-1)^n.
+// Floor/truncate need IEEE bits, so convert qf32 back to sf before them.
+static inline void hvx_vec_sincos_reduce_f32(HVX_Vector x, HVX_Vector * y, HVX_Vector * sign) {
+ HVX_Vector const_inv_pi = hvx_vec_splat_f32(0.3183098861837907f);
+ HVX_Vector const_half = hvx_vec_splat_f32(0.5f);
+ HVX_Vector const_pi = hvx_vec_splat_f32(3.141592653589793f);
+ HVX_Vector const_one = hvx_vec_splat_f32(1.0f);
HVX_Vector const_neg_one = hvx_vec_splat_f32(-1.0f);
+ HVX_Vector const_one_i = Q6_V_vsplat_R(1);
+
+ HVX_Vector x_over_pi = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(x, const_inv_pi));
+ x_over_pi = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(x_over_pi, const_half));
- // n = floor(x * (1/pi) + 0.5)
- HVX_Vector n_float = hvx_vec_floor_f32(hvx_vec_add_f32_f32(hvx_vec_mul_f32_f32(x, const_inv_pi), const_half));
+ HVX_Vector n_float = hvx_vec_floor_f32(x_over_pi);
+ HVX_Vector n_int = hvx_vec_truncate_f32(n_float);
- // y = x - n * pi
- HVX_Vector y = hvx_vec_sub_f32_f32(x, hvx_vec_mul_f32_f32(n_float, const_pi));
+ HVX_Vector n_pi = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(n_float, const_pi));
+ *y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(x, n_pi));
- // Sign determination: if n is odd, sign is -1.0f, else 1.0f
- // half_n = n * 0.5f
- HVX_Vector half_n = hvx_vec_mul_f32_f32(n_float, const_half);
- // floor_half_n = floor(half_n)
- HVX_Vector floor_half_n = hvx_vec_floor_f32(half_n);
- // is_odd = half_n > floor_half_n
- HVX_VectorPred is_odd = Q6_Q_vcmp_gt_VsfVsf(half_n, floor_half_n);
- // sign = vmux(is_odd, -1.0f, 1.0f)
- HVX_Vector sign = Q6_V_vmux_QVV(is_odd, const_neg_one, const_one);
+ HVX_VectorPred is_odd = Q6_Q_vcmp_eq_VwVw(Q6_V_vand_VV(n_int, const_one_i), const_one_i);
+ *sign = Q6_V_vmux_QVV(is_odd, const_neg_one, const_one);
+}
- // z = y^2
- HVX_Vector z = hvx_vec_mul_f32_f32(y, y);
+static inline void hvx_vec_sincos_f32(HVX_Vector x, HVX_Vector * vcos, HVX_Vector * vsin) {
+ HVX_Vector y;
+ HVX_Vector sign;
+ hvx_vec_sincos_reduce_f32(x, &y, &sign);
+
+ HVX_Vector z = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(y, y));
- // Chebyshev approximation for cos(y)
HVX_Vector c4 = hvx_vec_splat_f32(2.3557242013849433e-05f);
HVX_Vector c3 = hvx_vec_splat_f32(-0.0013871428263450528f);
HVX_Vector c2 = hvx_vec_splat_f32(0.041665895266688284f);
HVX_Vector c1 = hvx_vec_splat_f32(-0.4999999360426369f);
HVX_Vector c0 = hvx_vec_splat_f32(0.9999999999071725f);
- HVX_Vector cos_y = hvx_vec_add_f32_f32(c3, hvx_vec_mul_f32_f32(z, c4));
- cos_y = hvx_vec_add_f32_f32(c2, hvx_vec_mul_f32_f32(z, cos_y));
- cos_y = hvx_vec_add_f32_f32(c1, hvx_vec_mul_f32_f32(z, cos_y));
- cos_y = hvx_vec_add_f32_f32(c0, hvx_vec_mul_f32_f32(z, cos_y));
-
- return hvx_vec_mul_f32_f32(cos_y, sign);
-}
-
-static inline HVX_Vector hvx_vec_sin_f32(HVX_Vector x) {
- HVX_Vector const_inv_pi = hvx_vec_splat_f32(0.3183098861837907f);
- HVX_Vector const_half = hvx_vec_splat_f32(0.5f);
- HVX_Vector const_pi = hvx_vec_splat_f32(3.141592653589793f);
- HVX_Vector const_one = hvx_vec_splat_f32(1.0f);
- HVX_Vector const_neg_one = hvx_vec_splat_f32(-1.0f);
-
- // n = floor(x * (1/pi) + 0.5)
- HVX_Vector n_float = hvx_vec_floor_f32(hvx_vec_add_f32_f32(hvx_vec_mul_f32_f32(x, const_inv_pi), const_half));
+ HVX_Vector cos_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(c3, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, c4))));
+ cos_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(c2, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, cos_y))));
+ cos_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(c1, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, cos_y))));
+ cos_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(c0, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, cos_y))));
- // y = x - n * pi
- HVX_Vector y = hvx_vec_sub_f32_f32(x, hvx_vec_mul_f32_f32(n_float, const_pi));
-
- // Sign determination: if n is odd, sign is -1.0f, else 1.0f
- // half_n = n * 0.5f
- HVX_Vector half_n = hvx_vec_mul_f32_f32(n_float, const_half);
- // floor_half_n = floor(half_n)
- HVX_Vector floor_half_n = hvx_vec_floor_f32(half_n);
- // is_odd = half_n > floor_half_n
- HVX_VectorPred is_odd = Q6_Q_vcmp_gt_VsfVsf(half_n, floor_half_n);
- // sign = vmux(is_odd, -1.0f, 1.0f)
- HVX_Vector sign = Q6_V_vmux_QVV(is_odd, const_neg_one, const_one);
-
- // z = y^2
- HVX_Vector z = hvx_vec_mul_f32_f32(y, y);
-
- // Chebyshev approximation for sin(y)
HVX_Vector s4 = hvx_vec_splat_f32(2.642186986152672e-06f);
HVX_Vector s3 = hvx_vec_splat_f32(-0.00019825318964070864f);
HVX_Vector s2 = hvx_vec_splat_f32(0.00833326283319605f);
HVX_Vector s1 = hvx_vec_splat_f32(-0.16666666082087775f);
HVX_Vector s0 = hvx_vec_splat_f32(0.999999999915155f);
- HVX_Vector sin_y = hvx_vec_add_f32_f32(s3, hvx_vec_mul_f32_f32(z, s4));
- sin_y = hvx_vec_add_f32_f32(s2, hvx_vec_mul_f32_f32(z, sin_y));
- sin_y = hvx_vec_add_f32_f32(s1, hvx_vec_mul_f32_f32(z, sin_y));
- sin_y = hvx_vec_add_f32_f32(s0, hvx_vec_mul_f32_f32(z, sin_y));
- sin_y = hvx_vec_mul_f32_f32(y, sin_y);
+ HVX_Vector sin_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(s3, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, s4))));
+ sin_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(s2, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, sin_y))));
+ sin_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(s1, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, sin_y))));
+ sin_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(s0, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, sin_y))));
+ sin_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(y, sin_y));
+
+ *vcos = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(cos_y, sign));
+ *vsin = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(sin_y, sign));
+}
- return hvx_vec_mul_f32_f32(sin_y, sign);
+static inline HVX_Vector hvx_vec_cos_f32(HVX_Vector x) {
+ HVX_Vector vcos;
+ HVX_Vector vsin;
+ hvx_vec_sincos_f32(x, &vcos, &vsin);
+ return vcos;
+}
+
+static inline HVX_Vector hvx_vec_sin_f32(HVX_Vector x) {
+ HVX_Vector vcos;
+ HVX_Vector vsin;
+ hvx_vec_sincos_f32(x, &vcos, &vsin);
+ return vsin;
}
#endif /* HVX_SIN_COS_H */
diff --git a/ggml/src/ggml-hexagon/htp/rope-ops.c b/ggml/src/ggml-hexagon/htp/rope-ops.c
index 6c6898249..0a4b31ccb 100644
--- a/ggml/src/ggml-hexagon/htp/rope-ops.c
+++ b/ggml/src/ggml-hexagon/htp/rope-ops.c
@@ -17,8 +17,8 @@
#include "ggml-common.h"
#include "htp-ctx.h"
#include "htp-ops.h"
-#include "htp-ops.h"
#include "htp-tensor.h"
+#include "rope-ops.h"
// Redefined the rope type constants as we can't include ggml.h
#define HTP_ROPE_TYPE_NORMAL 0
@@ -27,9 +27,6 @@
#define HTP_ROPE_TYPE_VISION 24
#define HTP_ROPE_TYPE_IMROPE 40
-#define HTP_ROPE_SPAD_NROWS 16
-#define HTP_ROPE_SPAD_BLOCK (HTP_ROPE_SPAD_NROWS/2)
-
#define htp_rope_preamble \
const uint32_t ne00 = src0->ne[0]; \
const uint32_t ne01 = src0->ne[1]; \
@@ -65,26 +62,27 @@ struct htp_rope_context {
float beta_fast;
float beta_slow;
float theta_scale;
+ float theta_scale_32;
+ float theta_powers[32];
float corr_dims[2];
uint32_t src0_nrows_per_thread;
- size_t spad_stride;
struct htp_ops_context * octx;
+ uint8_t * vtcm_base;
+ size_t spad_per_thread;
+ size_t theta_cache_offset;
+
size_t src0_row_size;
size_t src0_row_stride;
size_t dst_row_size;
size_t dst_row_stride;
size_t src0_row_size_aligned;
- size_t dst_row_size_aligned;
- size_t theta_cache_offset;
uint32_t src0_nrows;
struct fastdiv_values div_ne2_ne1;
struct fastdiv_values div_ne1;
-
- uint64_t t_start;
};
static float rope_yarn_ramp(const float low, const float high, const int i0) {
@@ -112,94 +110,80 @@ static inline void rope_yarn_one(float theta, float freq_scale, float * corr_dim
mscale_final *= 1.0f + 0.1f * logf(1.0f / freq_scale);
}
- cache[i0 + 0] = cosf(theta_final) * mscale_final;
- cache[i0 + 1] = sinf(theta_final) * mscale_final;
+ const uint32_t b = i0 / 64;
+ const uint32_t k = (i0 % 64) / 2;
+ cache[b * 64 + k] = cosf(theta_final) * mscale_final;
+ cache[b * 64 + 32 + k] = sinf(theta_final) * mscale_final;
+}
+
+// 32 thetas -> 32 deinterleaved pairs [cos[32] | sin[32]] at cache[i0].
+static inline void rope_cache_hvx_32(float * cache, uint32_t i0,
+ HVX_Vector v_theta,
+ const float * freq_factors,
+ HVX_Vector v_freq_scale,
+ HVX_Vector v_mscale) {
+ if (freq_factors) {
+ HVX_Vector v_ff = hvx_vmemu(freq_factors + i0 / 2);
+ v_theta = hvx_vec_mul_f32_f32(v_theta, hvx_vec_inverse_f32(v_ff));
+ }
+
+ HVX_Vector v_theta_final = hvx_vec_mul_f32_f32(v_theta, v_freq_scale);
+ HVX_Vector vcos;
+ HVX_Vector vsin;
+ hvx_vec_sincos_f32(v_theta_final, &vcos, &vsin);
+ vcos = hvx_vec_mul_f32_f32(vcos, v_mscale);
+ vsin = hvx_vec_mul_f32_f32(vsin, v_mscale);
+
+ if (((uintptr_t) (cache + i0)) % 128 == 0) {
+ hvx_vmem(cache + i0 + 0) = vcos;
+ hvx_vmem(cache + i0 + 32) = vsin;
+ } else {
+ hvx_vec_store_u(cache + i0 + 0, 32 * sizeof(float), vcos);
+ hvx_vec_store_u(cache + i0 + 32, 32 * sizeof(float), vsin);
+ }
}
static __attribute__((noinline)) void rope_cache_init(const float theta_base,
const float freq_scale,
const float * freq_factors,
float * corr_dims,
- const uint32_t ne0,
+ const uint32_t n_cache,
const float ext_factor,
const float mscale,
float * cache,
- const float theta_scale) {
+ const float theta_scale,
+ const float * theta_powers,
+ const float theta_scale_32) {
// ref: https://github.com/jquesnelle/yarn/blob/master/scaled_rope/LlamaYaRNScaledRotaryEmbedding.py
-#if __HVX_ARCH__ >= 79
- const bool is_v79_or_newer = true;
-#else
- const bool is_v79_or_newer = false;
-#endif
-
- if (is_v79_or_newer && ext_factor == 0.0f) {
+ if (ext_factor == 0.0f) {
// Fast path: fully vectorized
// We process 32 pairs (64 elements) per iteration.
- const uint32_t n_blocks = ne0 / 64;
-
- // Initialize theta scale powers: [1.0f, theta_scale, theta_scale^2, ..., theta_scale^31]
- float __attribute__((aligned(128))) theta_powers[32];
- theta_powers[0] = 1.0f;
- for (int j = 1; j < 32; j++) {
- theta_powers[j] = theta_powers[j - 1] * theta_scale;
- }
- HVX_Vector v_theta_powers = hvx_vmem(theta_powers);
+ const uint32_t n_blocks = n_cache / 64;
+ HVX_Vector v_theta_powers = hvx_vmemu(theta_powers);
HVX_Vector v_freq_scale = hvx_vec_splat_f32(freq_scale);
HVX_Vector v_mscale = hvx_vec_splat_f32(mscale);
- // Base theta starts at theta_base
float theta_block = theta_base;
- // The scale factor for the next block is theta_scale^32
- float theta_scale_32 = 1.0f;
- for (int j = 0; j < 32; j++) {
- theta_scale_32 *= theta_scale;
- }
for (uint32_t b = 0; b < n_blocks; b++) {
uint32_t i0 = b * 64;
HVX_Vector v_theta_base = hvx_vec_splat_f32(theta_block);
HVX_Vector v_theta = hvx_vec_mul_f32_f32(v_theta_base, v_theta_powers);
-
- if (freq_factors) {
- // Load 32 elements of freq_factors
- HVX_Vector v_ff = hvx_vmemu(freq_factors + i0 / 2);
- HVX_Vector v_inv_ff = hvx_vec_inverse_f32(v_ff);
- v_theta = hvx_vec_mul_f32_f32(v_theta, v_inv_ff);
- }
-
- HVX_Vector v_theta_final = hvx_vec_mul_f32_f32(v_theta, v_freq_scale);
-
- HVX_Vector vcos = hvx_vec_cos_f32(v_theta_final);
- HVX_Vector vsin = hvx_vec_sin_f32(v_theta_final);
-
- vcos = hvx_vec_mul_f32_f32(vcos, v_mscale);
- vsin = hvx_vec_mul_f32_f32(vsin, v_mscale);
-
- HVX_VectorPair vstore = Q6_W_vshuff_VVR(vsin, vcos, -4);
-
- if (((uintptr_t)cache) % 128 == 0) {
- hvx_vmem(cache + i0 + 0) = Q6_V_lo_W(vstore);
- hvx_vmem(cache + i0 + 32) = Q6_V_hi_W(vstore);
- } else {
- hvx_vec_store_u(cache + i0 + 0, 32 * sizeof(float), Q6_V_lo_W(vstore));
- hvx_vec_store_u(cache + i0 + 32, 32 * sizeof(float), Q6_V_hi_W(vstore));
- }
-
+ rope_cache_hvx_32(cache, i0, v_theta, freq_factors, v_freq_scale, v_mscale);
theta_block *= theta_scale_32;
}
// Leftovers
float theta = theta_block;
- for (uint32_t i0 = n_blocks * 64; i0 < ne0; i0 += 2) {
+ for (uint32_t i0 = n_blocks * 64; i0 < n_cache; i0 += 2) {
const float ff = freq_factors ? freq_factors[i0 / 2] : 1.0f;
rope_yarn_one(theta / ff, freq_scale, corr_dims, i0, ext_factor, mscale, cache);
theta *= theta_scale;
}
} else {
- // Fallback to original scalar loop
float theta = theta_base;
- for (uint32_t i0 = 0; i0 < ne0; i0 += 2) {
+ for (uint32_t i0 = 0; i0 < n_cache; i0 += 2) {
const float ff = freq_factors ? freq_factors[i0 / 2] : 1.0f;
rope_yarn_one(theta / ff, freq_scale, corr_dims, i0, ext_factor, mscale, cache);
theta *= theta_scale;
@@ -207,6 +191,72 @@ static __attribute__((noinline)) void rope_cache_init(const float theta_base,
}
}
+static inline float mrope_pick_theta(float theta_t, float theta_h, float theta_w, float theta_e,
+ int sector, const int32_t sections[4], int sec_w, int sec_e,
+ bool is_imrope) {
+ if (is_imrope) {
+ if (sector % 3 == 0 && sector < 3 * sections[0]) { return theta_t; }
+ else if (sector % 3 == 1 && sector < 3 * sections[1]) { return theta_h; }
+ else if (sector % 3 == 2 && sector < 3 * sections[2]) { return theta_w; }
+ else { return theta_e; }
+ }
+ if (sector < sections[0]) { return theta_t; }
+ else if (sector < sec_w) { return theta_h; }
+ else if (sector < sec_e) { return theta_w; }
+ else { return theta_e; }
+}
+
+// lane j is 1 when (j % 3) == rem
+static const float __attribute__((aligned(128))) mrope_mod3_eq0[32] = {
+ 1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0
+};
+static const float __attribute__((aligned(128))) mrope_mod3_eq1[32] = {
+ 0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1
+};
+static const float __attribute__((aligned(128))) mrope_mod3_eq2[32] = {
+ 0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0
+};
+
+static const float __attribute__((aligned(128))) mrope_k_ramp[32] = {
+ 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,
+ 16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31
+};
+
+static inline HVX_VectorPred mrope_mask_eq1(const float * m) {
+ return Q6_Q_vcmp_gt_VsfVsf(hvx_vmemu(m), Q6_V_vzero());
+}
+
+// IMROPE without wrap: theta[k] = pos[k % 3] * scale^k
+static inline HVX_Vector mrope_thetas_imrope_mod3(float pos_t, float pos_h, float pos_w,
+ uint32_t k0, HVX_Vector v_powers, float scale_block) {
+ const int r = (int) (k0 % 3);
+ const float * mt = (r == 0) ? mrope_mod3_eq0 : (r == 1) ? mrope_mod3_eq2 : mrope_mod3_eq1;
+ const float * mh = (r == 0) ? mrope_mod3_eq1 : (r == 1) ? mrope_mod3_eq0 : mrope_mod3_eq2;
+
+ HVX_Vector v = hvx_vec_splat_f32(pos_w);
+ v = Q6_V_vmux_QVV(mrope_mask_eq1(mh), hvx_vec_splat_f32(pos_h), v);
+ v = Q6_V_vmux_QVV(mrope_mask_eq1(mt), hvx_vec_splat_f32(pos_t), v);
+ v = hvx_vec_mul_f32_f32(v, v_powers);
+ return hvx_vec_mul_f32_f32(v, hvx_vec_splat_f32(scale_block));
+}
+
+// Contiguous MROPE without wrap: theta[k] = pos[section(k)] * scale^k
+static inline HVX_Vector mrope_thetas_contig(float pos_t, float pos_h, float pos_w, float pos_e,
+ uint32_t k0, int s0, int sec_w, int sec_e,
+ HVX_Vector v_powers, float scale_block) {
+ HVX_Vector v_k = hvx_vec_add_f32_f32(hvx_vec_splat_f32((float) k0), hvx_vmemu(mrope_k_ramp));
+ HVX_VectorPred lt_s0 = Q6_Q_vcmp_gt_VsfVsf(hvx_vec_splat_f32((float) s0), v_k);
+ HVX_VectorPred lt_sw = Q6_Q_vcmp_gt_VsfVsf(hvx_vec_splat_f32((float) sec_w), v_k);
+ HVX_VectorPred lt_se = Q6_Q_vcmp_gt_VsfVsf(hvx_vec_splat_f32((float) sec_e), v_k);
+
+ HVX_Vector v = hvx_vec_splat_f32(pos_e);
+ v = Q6_V_vmux_QVV(lt_se, hvx_vec_splat_f32(pos_w), v);
+ v = Q6_V_vmux_QVV(lt_sw, hvx_vec_splat_f32(pos_h), v);
+ v = Q6_V_vmux_QVV(lt_s0, hvx_vec_splat_f32(pos_t), v);
+ v = hvx_vec_mul_f32_f32(v, v_powers);
+ return hvx_vec_mul_f32_f32(v, hvx_vec_splat_f32(scale_block));
+}
+
// pos_t/h/w/e: the four position ids for this sequence step (t=time, h=height, w=width, e=extra).
// sections[4]: number of head dims assigned to each position component.
static __attribute__((noinline)) void mrope_cache_init(const float pos_t,
@@ -219,23 +269,71 @@ static __attribute__((noinline)) void mrope_cache_init(const float pos_t,
const float freq_scale,
const float * freq_factors,
float * corr_dims,
- const uint32_t ne0,
+ const uint32_t n_cache,
const float ext_factor,
const float mscale,
float * cache,
- const float theta_scale) {
+ const float theta_scale,
+ const float * theta_powers,
+ const float theta_scale_32) {
const int sect_dims = sections[0] + sections[1] + sections[2] + sections[3];
const int sec_w = sections[0] + sections[1];
const int sec_e = sec_w + sections[2];
+ const uint32_t n_pairs = n_cache / 2;
+
+ const bool no_wrap = (sect_dims > 0) && (n_pairs <= (uint32_t) sect_dims);
+ const bool imrope_mod3 = is_imrope && !indep_sects && no_wrap
+ && sections[0] > 0 && sections[1] > 0 && sections[2] > 0
+ && n_pairs <= (uint32_t) (3 * sections[0])
+ && n_pairs <= (uint32_t) (3 * sections[1])
+ && n_pairs <= (uint32_t) (3 * sections[2]);
+ const bool contig = !is_imrope && !indep_sects && no_wrap;
+
+ if (ext_factor == 0.0f && (imrope_mod3 || contig)) {
+ HVX_Vector v_powers = hvx_vmemu(theta_powers);
+ HVX_Vector v_freq_scale = hvx_vec_splat_f32(freq_scale);
+ HVX_Vector v_mscale = hvx_vec_splat_f32(mscale);
+ float scale_block = 1.0f;
+ const uint32_t n_blocks = n_cache / 64;
+
+ for (uint32_t b = 0; b < n_blocks; b++) {
+ const uint32_t i0 = b * 64;
+ const uint32_t k0 = b * 32;
+ HVX_Vector v_theta = imrope_mod3
+ ? mrope_thetas_imrope_mod3(pos_t, pos_h, pos_w, k0, v_powers, scale_block)
+ : mrope_thetas_contig(pos_t, pos_h, pos_w, pos_e, k0, sections[0], sec_w, sec_e,
+ v_powers, scale_block);
+ rope_cache_hvx_32(cache, i0, v_theta, freq_factors, v_freq_scale, v_mscale);
+ scale_block *= theta_scale_32;
+ }
+
+ float theta_k = scale_block;
+ for (uint32_t k = n_blocks * 32; k < n_pairs; k++) {
+ const uint32_t i0 = 2 * k;
+ const float pos = mrope_pick_theta(pos_t, pos_h, pos_w, pos_e,
+ (int) k, sections, sec_w, sec_e, is_imrope);
+ const float ff = freq_factors ? freq_factors[k] : 1.0f;
+ rope_yarn_one(pos * theta_k / ff, freq_scale, corr_dims, i0, ext_factor, mscale, cache);
+ theta_k *= theta_scale;
+ }
+ return;
+ }
float theta_t = pos_t;
float theta_h = pos_h;
float theta_w = pos_w;
float theta_e = pos_e;
- for (uint32_t i0 = 0; i0 < ne0; i0 += 2) {
- const float ff = freq_factors ? freq_factors[i0 / 2] : 1.0f;
- const int sector = (i0 / 2) % sect_dims;
+ const bool use_hvx = (ext_factor == 0.0f);
+ float __attribute__((aligned(128))) thetas[32];
+ uint32_t n_thetas = 0;
+ uint32_t block_i0 = 0;
+
+ HVX_Vector v_freq_scale = hvx_vec_splat_f32(freq_scale);
+ HVX_Vector v_mscale = hvx_vec_splat_f32(mscale);
+
+ for (uint32_t i0 = 0; i0 < n_cache; i0 += 2) {
+ const int sector = (i0 / 2) % sect_dims;
if (indep_sects) {
// Reset theta when crossing into a new section.
@@ -245,28 +343,34 @@ static __attribute__((noinline)) void mrope_cache_init(const float pos_t,
else if (sector == sec_e) { theta_e = pos_e; }
}
- float theta;
- if (is_imrope) {
- // Interleaved: sector mod 3 selects component
- if (sector % 3 == 0 && sector < 3 * sections[0]) { theta = theta_t; }
- else if (sector % 3 == 1 && sector < 3 * sections[1]) { theta = theta_h; }
- else if (sector % 3 == 2 && sector < 3 * sections[2]) { theta = theta_w; }
- else { theta = theta_e; }
+ const float theta = mrope_pick_theta(theta_t, theta_h, theta_w, theta_e,
+ sector, sections, sec_w, sec_e, is_imrope);
+
+ if (use_hvx) {
+ if (n_thetas == 0) {
+ block_i0 = i0;
+ }
+ thetas[n_thetas++] = theta;
+ if (n_thetas == 32) {
+ rope_cache_hvx_32(cache, block_i0, hvx_vmemu(thetas), freq_factors, v_freq_scale, v_mscale);
+ n_thetas = 0;
+ }
} else {
- // Contiguous sections
- if (sector < sections[0]) { theta = theta_t; }
- else if (sector < sec_w) { theta = theta_h; }
- else if (sector < sec_e) { theta = theta_w; }
- else { theta = theta_e; }
+ const float ff = freq_factors ? freq_factors[i0 / 2] : 1.0f;
+ rope_yarn_one(theta / ff, freq_scale, corr_dims, i0, ext_factor, mscale, cache);
}
- rope_yarn_one(theta / ff, freq_scale, corr_dims, i0, ext_factor, mscale, cache);
-
theta_t *= theta_scale;
theta_h *= theta_scale;
theta_w *= theta_scale;
theta_e *= theta_scale;
}
+
+ for (uint32_t k = 0; k < n_thetas; k++) {
+ const uint32_t i0 = block_i0 + 2 * k;
+ const float ff = freq_factors ? freq_factors[i0 / 2] : 1.0f;
+ rope_yarn_one(thetas[k] / ff, freq_scale, corr_dims, i0, ext_factor, mscale, cache);
+ }
}
#define M_PI 3.1415926535897932384626433
@@ -283,52 +387,54 @@ static void rope_corr_dims(int n_dims,
dims[1] = MIN(n_dims - 1, end);
}
+static inline void hvx_rope_neox_mul(HVX_Vector v0, HVX_Vector v1, HVX_Vector vcos, HVX_Vector vsin,
+ HVX_Vector * o0, HVX_Vector * o1) {
+ HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(v0, vcos);
+ HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(v0, vsin);
+ HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(v1, vcos);
+ HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(v1, vsin);
+ *o0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(vx0_c, vx1_s));
+ *o1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(vx0_s, vx1_c));
+}
+
+// theta_cache full 32-pair blocks are deinterleaved [cos | sin].
static inline void hvx_rope_neox_f32_aa(float * restrict dst, const float * restrict src0, uint32_t ne, const float * restrict theta_cache) {
const uint32_t he = ne / 2;
const uint32_t nvec = he / 32;
const uint32_t nloe = he % 32;
- for (uint32_t i = 0; i < nvec; i++) {
- HVX_Vector v0 = ((const HVX_Vector *) src0)[i];
- HVX_Vector v1 = hvx_vmemu(src0 + he + i * 32);
-
- HVX_Vector v2 = ((const HVX_Vector *) theta_cache)[i * 2 + 0];
- HVX_Vector v3 = ((const HVX_Vector *) theta_cache)[i * 2 + 1];
-
- HVX_VectorPair vcos_sin = Q6_W_vdeal_VVR(v3, v2, -4);
-
- HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(v0, Q6_V_lo_W(vcos_sin));
- HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(v0, Q6_V_hi_W(vcos_sin));
- HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(v1, Q6_V_lo_W(vcos_sin));
- HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(v1, Q6_V_hi_W(vcos_sin));
-
- HVX_Vector v4 = Q6_Vqf32_vsub_Vqf32Vqf32(vx0_c, vx1_s);
- HVX_Vector v5 = Q6_Vqf32_vadd_Vqf32Vqf32(vx0_s, vx1_c);
-
- ((HVX_Vector *) dst)[i] = Q6_Vsf_equals_Vqf32(v4);
- hvx_vmemu(dst + he + i * 32) = Q6_Vsf_equals_Vqf32(v5);
+ if (nloe == 0) {
+ const HVX_Vector * vs = (const HVX_Vector *) src0;
+ const HVX_Vector * vt = (const HVX_Vector *) theta_cache;
+ HVX_Vector * vd = (HVX_Vector *) dst;
+ for (uint32_t i = 0; i < nvec; i++) {
+ HVX_Vector o0, o1;
+ hvx_rope_neox_mul(vs[i], vs[nvec + i], vt[i * 2 + 0], vt[i * 2 + 1], &o0, &o1);
+ vd[i] = o0;
+ vd[nvec + i] = o1;
+ }
+ return;
}
- if (nloe > 0) {
- HVX_Vector v0 = hvx_vmemu(src0 + nvec * 32);
- HVX_Vector v1 = hvx_vmemu(src0 + he + nvec * 32);
-
- HVX_Vector v2 = ((const HVX_Vector *) theta_cache)[nvec * 2 + 0];
- HVX_Vector v3 = ((const HVX_Vector *) theta_cache)[nvec * 2 + 1];
-
- HVX_VectorPair vcos_sin = Q6_W_vdeal_VVR(v3, v2, -4);
-
- HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(v0, Q6_V_lo_W(vcos_sin));
- HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(v0, Q6_V_hi_W(vcos_sin));
- HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(v1, Q6_V_lo_W(vcos_sin));
- HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(v1, Q6_V_hi_W(vcos_sin));
-
- HVX_Vector v4 = Q6_Vqf32_vsub_Vqf32Vqf32(vx0_c, vx1_s);
- HVX_Vector v5 = Q6_Vqf32_vadd_Vqf32Vqf32(vx0_s, vx1_c);
-
- hvx_vec_store_u(dst + nvec * 32, nloe * sizeof(float), Q6_Vsf_equals_Vqf32(v4));
- hvx_vec_store_u(dst + he + nvec * 32, nloe * sizeof(float), Q6_Vsf_equals_Vqf32(v5));
+ for (uint32_t i = 0; i < nvec; i++) {
+ HVX_Vector o0, o1;
+ hvx_rope_neox_mul(((const HVX_Vector *) src0)[i],
+ hvx_vmemu(src0 + he + i * 32),
+ ((const HVX_Vector *) theta_cache)[i * 2 + 0],
+ ((const HVX_Vector *) theta_cache)[i * 2 + 1],
+ &o0, &o1);
+ ((HVX_Vector *) dst)[i] = o0;
+ hvx_vmemu(dst + he + i * 32) = o1;
}
+
+ HVX_Vector v0 = hvx_vmemu(src0 + nvec * 32);
+ HVX_Vector v1 = hvx_vmemu(src0 + he + nvec * 32);
+ HVX_Vector vcos = hvx_vmemu(theta_cache + nvec * 64);
+ HVX_Vector vsin = hvx_vmemu(theta_cache + nvec * 64 + 32);
+ HVX_Vector o0, o1;
+ hvx_rope_neox_mul(v0, v1, vcos, vsin, &o0, &o1);
+ hvx_vec_store_u(dst + nvec * 32, nloe * sizeof(float), o0);
+ hvx_vec_store_u(dst + he + nvec * 32, nloe * sizeof(float), o1);
}
static inline void hvx_rope_f32_aa(float * restrict dst, const float * restrict src0, uint32_t ne, const float * restrict theta_cache) {
@@ -339,16 +445,15 @@ static inline void hvx_rope_f32_aa(float * restrict dst, const float * restrict
HVX_Vector v0 = ((const HVX_Vector *) src0)[i * 2 + 0];
HVX_Vector v1 = ((const HVX_Vector *) src0)[i * 2 + 1];
- HVX_Vector v2 = ((const HVX_Vector *) theta_cache)[i * 2 + 0];
- HVX_Vector v3 = ((const HVX_Vector *) theta_cache)[i * 2 + 1];
+ HVX_Vector vcos = ((const HVX_Vector *) theta_cache)[i * 2 + 0];
+ HVX_Vector vsin = ((const HVX_Vector *) theta_cache)[i * 2 + 1];
- HVX_VectorPair vx0_x1 = Q6_W_vdeal_VVR(v1, v0, -4);
- HVX_VectorPair vcos_sin = Q6_W_vdeal_VVR(v3, v2, -4);
+ HVX_VectorPair vx0_x1 = Q6_W_vdeal_VVR(v1, v0, -4);
- HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), Q6_V_lo_W(vcos_sin));
- HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), Q6_V_hi_W(vcos_sin));
- HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), Q6_V_lo_W(vcos_sin));
- HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), Q6_V_hi_W(vcos_sin));
+ HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), vcos);
+ HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), vsin);
+ HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), vcos);
+ HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), vsin);
HVX_Vector v4 = Q6_Vqf32_vsub_Vqf32Vqf32(vx0_c, vx1_s);
HVX_Vector v5 = Q6_Vqf32_vadd_Vqf32Vqf32(vx0_s, vx1_c);
@@ -362,15 +467,15 @@ static inline void hvx_rope_f32_aa(float * restrict dst, const float * restrict
if (nloe > 0) {
if (nloe <= 32) {
HVX_Vector v0 = hvx_vmemu(src0 + nvec * 64);
- HVX_Vector v2 = hvx_vmemu(theta_cache + nvec * 64);
+ HVX_Vector vcos = hvx_vmemu(theta_cache + nvec * 64);
+ HVX_Vector vsin = hvx_vmemu(theta_cache + nvec * 64 + 32);
- HVX_VectorPair vx0_x1 = Q6_W_vdeal_VVR(Q6_V_vzero(), v0, -4);
- HVX_VectorPair vcos_sin = Q6_W_vdeal_VVR(Q6_V_vzero(), v2, -4);
+ HVX_VectorPair vx0_x1 = Q6_W_vdeal_VVR(Q6_V_vzero(), v0, -4);
- HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), Q6_V_lo_W(vcos_sin));
- HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), Q6_V_hi_W(vcos_sin));
- HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), Q6_V_lo_W(vcos_sin));
- HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), Q6_V_hi_W(vcos_sin));
+ HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), vcos);
+ HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), vsin);
+ HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), vcos);
+ HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), vsin);
HVX_Vector v4 = Q6_Vqf32_vsub_Vqf32Vqf32(vx0_c, vx1_s);
HVX_Vector v5 = Q6_Vqf32_vadd_Vqf32Vqf32(vx0_s, vx1_c);
@@ -382,16 +487,15 @@ static inline void hvx_rope_f32_aa(float * restrict dst, const float * restrict
HVX_Vector v0 = hvx_vmemu(src0 + nvec * 64);
HVX_Vector v1 = hvx_vmemu(src0 + nvec * 64 + 32);
- HVX_Vector v2 = hvx_vmemu(theta_cache + nvec * 64);
- HVX_Vector v3 = hvx_vmemu(theta_cache + nvec * 64 + 32);
+ HVX_Vector vcos = hvx_vmemu(theta_cache + nvec * 64);
+ HVX_Vector vsin = hvx_vmemu(theta_cache + nvec * 64 + 32);
- HVX_VectorPair vx0_x1 = Q6_W_vdeal_VVR(v1, v0, -4);
- HVX_VectorPair vcos_sin = Q6_W_vdeal_VVR(v3, v2, -4);
+ HVX_VectorPair vx0_x1 = Q6_W_vdeal_VVR(v1, v0, -4);
- HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), Q6_V_lo_W(vcos_sin));
- HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), Q6_V_hi_W(vcos_sin));
- HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), Q6_V_lo_W(vcos_sin));
- HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), Q6_V_hi_W(vcos_sin));
+ HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), vcos);
+ HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), vsin);
+ HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), vcos);
+ HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), vsin);
HVX_Vector v4 = Q6_Vqf32_vsub_Vqf32Vqf32(vx0_c, vx1_s);
HVX_Vector v5 = Q6_Vqf32_vadd_Vqf32Vqf32(vx0_s, vx1_c);
@@ -404,54 +508,23 @@ static inline void hvx_rope_f32_aa(float * restrict dst, const float * restrict
}
}
-static void inline rope_basic_f32(struct htp_rope_context * rctx, uint8_t * restrict dst, uint8_t * restrict src,
- uint32_t nr, uint32_t ne0, const float * restrict theta_cache) {
- const uint32_t n_offs = rctx->n_offs; // VLEN-aligned (enforced by supports_op)
- #pragma unroll(4)
- for (uint32_t i = 0; i < nr; i++) {
- float * d = (float *) (dst + i * rctx->dst_row_size_aligned);
- float * s = (float *) (src + i * rctx->src0_row_size_aligned);
-
- hvx_rope_f32_aa(d + n_offs, s + n_offs, rctx->n_dims, theta_cache);
-
- // fill the remain channels with data from src tensor
- if (n_offs > 0) {
- hvx_copy_f32_uu((uint8_t *) d, (uint8_t *) s, n_offs);
- }
- if (n_offs + rctx->n_dims < ne0) {
- hvx_copy_f32_uu((uint8_t *)(d + n_offs + rctx->n_dims), (uint8_t *)(s + n_offs + rctx->n_dims), ne0 - n_offs - rctx->n_dims);
- }
- }
-}
-
-static void inline rope_neox_f32(struct htp_rope_context * rctx, uint8_t * restrict dst, uint8_t * restrict src,
- uint32_t nr, uint32_t ne0, const float * restrict theta_cache) {
- const uint32_t n_offs = rctx->n_offs; // VLEN-aligned (enforced by supports_op)
+static void inline rope_basic_f32_inplace(struct htp_rope_context * rctx, uint8_t * src,
+ uint32_t nr, const float * restrict theta_cache) {
+ const uint32_t n_offs = rctx->n_offs;
#pragma unroll(4)
for (uint32_t i = 0; i < nr; i++) {
- float * d = (float *) (dst + i * rctx->dst_row_size_aligned);
float * s = (float *) (src + i * rctx->src0_row_size_aligned);
-
- hvx_rope_neox_f32_aa(d + n_offs, s + n_offs, rctx->n_dims, theta_cache);
-
- // fill the remain channels with data from src tensor
- if (n_offs > 0) {
- hvx_copy_f32_uu((uint8_t *) d, (uint8_t *) s, n_offs);
- }
- if (n_offs + rctx->n_dims < ne0) {
- hvx_copy_f32_uu((uint8_t *)(d + n_offs + rctx->n_dims), (uint8_t *)(s + n_offs + rctx->n_dims), ne0 - n_offs - rctx->n_dims);
- }
+ hvx_rope_f32_aa(s + n_offs, s + n_offs, rctx->n_dims, theta_cache);
}
}
-static void inline rope_vision_f32(struct htp_rope_context * rctx, uint8_t * restrict dst, uint8_t * restrict src,
- uint32_t nr, uint32_t ne0, const float * restrict theta_cache) {
+static void inline rope_neox_f32_inplace(struct htp_rope_context * rctx, uint8_t * src,
+ uint32_t nr, uint32_t ne, const float * restrict theta_cache) {
+ const uint32_t n_offs = rctx->n_offs;
#pragma unroll(4)
for (uint32_t i = 0; i < nr; i++) {
- float * d = (float *) (dst + i * rctx->dst_row_size_aligned);
float * s = (float *) (src + i * rctx->src0_row_size_aligned);
-
- hvx_rope_neox_f32_aa(d, s, ne0, theta_cache);
+ hvx_rope_neox_f32_aa(s + n_offs, s + n_offs, ne, theta_cache);
}
}
@@ -477,20 +550,18 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) {
return;
}
- uint64_t tt = HAP_perf_get_qtimer_count();
-
const int32_t mode = rctx->mode;
// MROPE, IMROPE and VISION use NEOX-style pairing for the rotation
const bool is_neox = (mode & HTP_ROPE_TYPE_NEOX) || (mode & HTP_ROPE_TYPE_MROPE);
const bool is_vision = (mode == HTP_ROPE_TYPE_VISION);
// VTCM setup
- uint8_t * src0_spad_base = octx->src0_spad.data + (ith * octx->src0_spad.size_per_thread);
+ uint8_t * src0_spad_base = rctx->vtcm_base + (ith * rctx->spad_per_thread);
float * theta_cache = (float *) (src0_spad_base);
src0_spad_base = src0_spad_base + rctx->theta_cache_offset;
- uint8_t * dst_spad_base = octx->dst_spad.data + (ith * octx->dst_spad.size_per_thread);
dma_queue * dma_queue = octx->ctx->dma[ith];
+ struct htp_thread_trace * tr = &octx->ctx->trace[ith];
const int32_t * pos = (const int32_t *) src1->data;
const float * freq_factors = src2 ? (const float *) src2->data : NULL;
@@ -501,6 +572,7 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) {
uint32_t ir = src0_start_row;
uint32_t prev_i2 = (uint32_t) -1;
+ uint32_t cur_slot = 0;
for (uint32_t i3 = i3_start; i3 < ne3; i3++) { // batch
const uint32_t i2_init = (i3 == i3_start) ? i2_start : 0;
@@ -513,35 +585,30 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) {
const uint32_t nrows = MIN(src0_end_row - ir, ne1 - i1);
// Depth before prefetch
- uint32_t dma_depth = dma_queue_depth(dma_queue);
-
- // FARF(HIGH, "rope-block %u: ir %u n-rows %u dma-depth %u : usec %u", ith, ir, nrows, dma_depth,
- // (unsigned) HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - rctx->t_start));
+ const uint32_t dma_depth = dma_queue_depth(dma_queue);
- // Prefetch loop
- for (uint32_t pnr = 0, pr = 0; pr < nrows && pr < HTP_ROPE_SPAD_NROWS; pr += pnr) {
- pnr = MIN(nrows - pr, HTP_ROPE_SPAD_BLOCK);
+ // Prefetch up to 2 blocks
+ const uint32_t p_nrows = MIN(nrows, 2 * HTP_ROPE_SPAD_BLOCK);
+ for (uint32_t pr = 0; pr < p_nrows; pr += HTP_ROPE_SPAD_BLOCK) {
+ const uint32_t pnr = MIN(nrows - pr, HTP_ROPE_SPAD_BLOCK);
+ const uint32_t slot = (cur_slot + pr / HTP_ROPE_SPAD_BLOCK) % HTP_ROPE_SPAD_NSLOTS;
+ uint8_t * spad_slot = rope_spad_slot(src0_spad_base, slot, rctx->src0_row_size_aligned);
+ const uint8_t * src_addr = (const uint8_t *) src0->data + i3 * nb03 + i2 * nb02 + (i1 + pr) * nb01;
- uint32_t pi1 = i1 + pr;
- uint32_t pir = ir + pr;
+ // Dummy DMA transaction for sequencing (interleaving wr, rd, wr, rd, ...)
+ dma_queue_push(dma_queue, dma_make_ptr((void *) dst->data, spad_slot), 0, 0, 0, 0);
- // Dummy DMA transaction for sequencing (interleaving dst,src,dst,...)
- dma_queue_push_vtcm_to_ddr(dma_queue, dma_make_ptr((void *) dst->data, dst_spad_base + pr * rctx->dst_row_size_aligned), 0, 0, 0);
-
- const uint8_t * src_addr = (const uint8_t *) src0->data + i3 * nb03 + i2 * nb02 + pi1 * nb01;
- uint8_t * src_spad = src0_spad_base + pr * rctx->src0_row_size_aligned;
-
- // Copy only the row payload while striding the DDR source
- dma_queue_push(dma_queue, dma_make_ptr(src_spad, src_addr),
+ dma_queue_push(dma_queue, dma_make_ptr(spad_slot, src_addr),
rctx->src0_row_size_aligned, rctx->src0_row_stride, rctx->src0_row_size, pnr);
-
- // FARF(HIGH, "rope-prefetch %u: pr %u i1 %u i2 %u i3 %u src-spad %p src-addr %p pnr %u", ith, pir, pi1, i2, i3, src_spad, src_addr, pnr);
}
// Update theta cache
if (i2 != prev_i2) {
prev_i2 = i2;
+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, i2);
+ // VISION rotates the full row; other modes only rotate n_dims.
+ const uint32_t n_cache = is_vision ? ne0 : (uint32_t) rctx->n_dims;
const bool is_mrope = (rctx->mode & HTP_ROPE_TYPE_MROPE) != 0;
if (is_mrope) {
// src1 holds four position arrays stacked along ne0:
@@ -554,66 +621,71 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) {
(float) pos[i2 + ne2 * 3],
rctx->sections, is_imrope, is_vision,
rctx->freq_scale, freq_factors, rctx->corr_dims,
- ne0, rctx->ext_factor, rctx->attn_factor,
- theta_cache, rctx->theta_scale);
+ n_cache, rctx->ext_factor, rctx->attn_factor,
+ theta_cache, rctx->theta_scale, rctx->theta_powers, rctx->theta_scale_32);
} else {
rope_cache_init(pos[i2], rctx->freq_scale, freq_factors, rctx->corr_dims,
- ne0, rctx->ext_factor, rctx->attn_factor,
- theta_cache, rctx->theta_scale);
+ n_cache, rctx->ext_factor, rctx->attn_factor,
+ theta_cache, rctx->theta_scale, rctx->theta_powers, rctx->theta_scale_32);
}
+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, i2);
}
// Skip output DMA transactions from prev block (if any)
- // No need to wait for those here since we're explicitly waiting for the latest prefecthes below.
- for (uint32_t d=0; d < dma_depth; d++) { dma_queue_pop_nowait(dma_queue); }
+ for (uint32_t d = 0; d < dma_depth; d++) { dma_queue_pop_nowait(dma_queue); }
// Compute loop
- for (uint32_t cnr = 0, cr = 0; cr < nrows; cr += cnr, ir += cnr, i1 += cnr) {
- // Number of rows to compute
- cnr = MIN(nrows - cr, HTP_ROPE_SPAD_BLOCK);
+ const uint32_t ne = is_vision ? ne0 : rctx->n_dims;
+ const uint32_t base_i1 = i1;
+ const uint32_t base_ir = ir;
- uint8_t * dst_spad = (uint8_t *) dma_queue_pop(dma_queue).src;
- uint8_t * src_spad = (uint8_t *) dma_queue_pop(dma_queue).dst;
+ for (uint32_t cnr = 0, cr = 0; cr < nrows; cr += cnr) {
+ cnr = MIN(nrows - cr, HTP_ROPE_SPAD_BLOCK);
+ const uint32_t slot = (cur_slot + cr / HTP_ROPE_SPAD_BLOCK) % HTP_ROPE_SPAD_NSLOTS;
+ const uint32_t cur_ir = base_ir + cr;
+ const uint32_t cur_i1 = base_i1 + cr;
- // FARF(HIGH, "rope-compute %u: ir %u i1 %u i2 %u i3 %u src-spad %p cnr %u : usec %u", ith, ir, i1, i2, i3, src_spad, cnr,
- // (unsigned) HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - rctx->t_start));
+ dma_queue_pop(dma_queue);
+ uint8_t * cur_spad = (uint8_t *) dma_queue_pop(dma_queue).dst;
- if (is_vision) {
- rope_vision_f32(rctx, dst_spad, src_spad, cnr, ne0, theta_cache);
- } else if (is_neox) {
- rope_neox_f32(rctx, dst_spad, src_spad, cnr, ne0, theta_cache);
+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, cur_ir);
+ if (is_neox || is_vision) {
+ rope_neox_f32_inplace(rctx, cur_spad, cnr, ne, theta_cache);
} else {
- rope_basic_f32(rctx, dst_spad, src_spad, cnr, ne0, theta_cache);
+ rope_basic_f32_inplace(rctx, cur_spad, cnr, theta_cache);
}
+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, cur_ir);
- uint8_t * dst_addr = (uint8_t *) dst->data + i3 * nb3 + i2 * nb2 + i1 * nb1;
-
- // Write only the row payload while striding the DDR dst
- dma_queue_push(dma_queue, dma_make_ptr(dst_addr, dst_spad),
- rctx->dst_row_stride, rctx->dst_row_size_aligned, rctx->dst_row_size, cnr);
+ uint8_t * dst_addr = (uint8_t *) dst->data + i3 * nb3 + i2 * nb2 + cur_i1 * nb1;
+ dma_queue_push(dma_queue, dma_make_ptr(dst_addr, cur_spad),
+ rctx->dst_row_stride, rctx->src0_row_size_aligned, rctx->dst_row_size, cnr);
- // Prefetch more rows (if any)
- if ((cr + HTP_ROPE_SPAD_NROWS) < nrows) {
- uint32_t pnr = MIN(nrows - (cr + HTP_ROPE_SPAD_NROWS), HTP_ROPE_SPAD_BLOCK);
- uint32_t pi1 = i1 + HTP_ROPE_SPAD_NROWS;
- uint32_t pir = ir + HTP_ROPE_SPAD_NROWS;
+ // Prefetch 2 blocks ahead into the slot just freed
+ if ((cr + 2 * HTP_ROPE_SPAD_BLOCK) < nrows) {
+ const uint32_t p_cr = cr + 2 * HTP_ROPE_SPAD_BLOCK;
+ const uint32_t pnr = MIN(nrows - p_cr, HTP_ROPE_SPAD_BLOCK);
+ const uint32_t p_slot = (cur_slot + p_cr / HTP_ROPE_SPAD_BLOCK) % HTP_ROPE_SPAD_NSLOTS;
+ uint8_t * p_spad = rope_spad_slot(src0_spad_base, p_slot, rctx->src0_row_size_aligned);
+ const uint8_t * src_addr = (const uint8_t *) src0->data + i3 * nb03 + i2 * nb02 + (base_i1 + p_cr) * nb01;
- const uint8_t * src_addr = (const uint8_t *) src0->data + i3 * nb03 + i2 * nb02 + pi1 * nb01;
- dma_queue_push(dma_queue, dma_make_ptr(src_spad, src_addr),
+ dma_queue_push(dma_queue, dma_make_ptr(p_spad, src_addr),
rctx->src0_row_size_aligned, rctx->src0_row_stride, rctx->src0_row_size, pnr);
-
- // FARF(HIGH, "rope-prefetch %u: pr %u i1 %u i2 %u i3 %u src-spad %p src-addr %p pnr %u", ith, pir, pi1, i2, i3, src_spad, src_addr, pnr);
}
}
+
+ const uint32_t n_chunks = (nrows + HTP_ROPE_SPAD_BLOCK - 1) / HTP_ROPE_SPAD_BLOCK;
+ cur_slot = (cur_slot + n_chunks) % HTP_ROPE_SPAD_NSLOTS;
+
+ ir += nrows;
+ i1 += nrows;
}
}
}
done:
dma_queue_flush(dma_queue);
- tt = HAP_perf_get_qtimer_count() - tt;
- FARF(HIGH, "rope-f32: %d/%d: (%u:%u) usec %u\n", ith, nth, src0_start_row, src0_end_row, (unsigned) HAP_perf_qtimer_count_to_us(tt));
+ FARF(HIGH, "rope-f32: %d/%d: (%u:%u)\n", ith, nth, src0_start_row, src0_end_row);
}
static int execute_op_rope_f32(struct htp_ops_context * octx) {
@@ -624,8 +696,6 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) {
const struct htp_tensor * src2 = octx->src[2];
const struct htp_tensor * dst = octx->dst;
- const char * op_type = "rope-f32";
-
switch (octx->op) {
case HTP_OP_ROPE:
break;
@@ -635,48 +705,23 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) {
return HTP_STATUS_NO_SUPPORT;
}
- const uint32_t ne0 = dst->ne[0];
- const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
- const uint32_t n_threads = MIN(octx->n_threads, src0_nrows);
+ const struct htp_rope_kernel_params * kparams = (const struct htp_rope_kernel_params *) octx->kernel_params;
+ assert(kparams->n_threads > 0);
+ assert(octx->ctx->vtcm_size >= kparams->vtcm_size);
+ const uint32_t ne0 = dst->ne[0];
const size_t src0_row_size = src0->ne[0] * sizeof(float);
const size_t src0_row_stride = src0->nb[1];
const size_t dst_row_size = dst->ne[0] * sizeof(float);
const size_t dst_row_stride = dst->nb[1];
- // Aligned row sizes for VTCM
- const size_t src0_row_size_aligned = hex_round_up(src0_row_size, VLEN);
- const size_t dst_row_size_aligned = hex_round_up(dst_row_stride, VLEN);
- const size_t theta_cache_size_aligned = hex_round_up(src0->ne[0] * sizeof(float), 256);
-
- // Calculate spad sizes per thread
- size_t src0_spad_per_thread = theta_cache_size_aligned + HTP_ROPE_SPAD_NROWS * src0_row_size_aligned;
- size_t dst_spad_per_thread = HTP_ROPE_SPAD_NROWS * dst_row_size_aligned;
- size_t spad_per_thread = src0_spad_per_thread + dst_spad_per_thread;
-
- // Check if we fit in VTCM
- size_t total_vtcm_needed = spad_per_thread * n_threads;
- if (octx->ctx->vtcm_size < total_vtcm_needed) {
- FARF(ERROR, "%s : current VTCM reservation %zu is too small, needed %zu\n", op_type, octx->ctx->vtcm_size, total_vtcm_needed);
- return HTP_STATUS_VTCM_TOO_SMALL;
- }
-
- octx->src0_spad.size_per_thread = src0_spad_per_thread;
- octx->dst_spad.size_per_thread = dst_spad_per_thread;
- octx->src0_spad.size = n_threads * src0_spad_per_thread;
- octx->dst_spad.size = n_threads * dst_spad_per_thread;
- octx->src1_spad.size = 0;
-
- octx->src0_spad.data = octx->ctx->vtcm_base; octx->src0_spad.src = NULL;
- octx->src1_spad.data = NULL; octx->src1_spad.src = NULL;
- octx->dst_spad.data = octx->src0_spad.data + octx->src0_spad.size; octx->dst_spad.src = NULL;
-
struct htp_rope_context rctx;
memset(&rctx, 0, sizeof(struct htp_rope_context));
- rctx.t_start = HAP_perf_get_qtimer_count();
-
- rctx.octx = octx;
+ rctx.octx = octx;
+ rctx.vtcm_base = (uint8_t *) octx->ctx->vtcm_base;
+ rctx.spad_per_thread = kparams->spad_per_thread;
+ rctx.theta_cache_offset = kparams->theta_cache_offset;
const int32_t * op_params = &octx->op_params[0];
rctx.n_dims = ((const int32_t *) op_params)[1];
@@ -693,31 +738,29 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) {
memcpy(&rctx.sections, (int32_t *) op_params + 11, sizeof(int) * 4);
rctx.theta_scale = powf(rctx.freq_base, -2.0f / rctx.n_dims);
+ rctx.theta_powers[0] = 1.0f;
+ for (int j = 1; j < 32; j++) {
+ rctx.theta_powers[j] = rctx.theta_powers[j - 1] * rctx.theta_scale;
+ }
+ rctx.theta_scale_32 = rctx.theta_powers[31] * rctx.theta_scale;
rope_corr_dims(rctx.n_dims, rctx.n_ctx_orig, rctx.freq_base, rctx.beta_fast, rctx.beta_slow, rctx.corr_dims);
- rctx.src0_row_size = src0_row_size;
- rctx.src0_row_stride = src0_row_stride;
- rctx.dst_row_size = dst_row_size;
- rctx.dst_row_stride = dst_row_stride;
- rctx.src0_row_size_aligned = src0_row_size_aligned;
- rctx.dst_row_size_aligned = dst_row_size_aligned;
- rctx.theta_cache_offset = theta_cache_size_aligned;
+ rctx.src0_row_size = src0_row_size;
+ rctx.src0_row_stride = src0_row_stride;
+ rctx.dst_row_size = dst_row_size;
+ rctx.dst_row_stride = dst_row_stride;
+ rctx.src0_row_size_aligned = kparams->src0_row_size_aligned;
- rctx.src0_nrows = src0_nrows;
- rctx.src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads;
-
- if (src0_nrows > 0) {
- rctx.div_ne2_ne1 = init_fastdiv_values(dst->ne[2] * dst->ne[1]);
- rctx.div_ne1 = init_fastdiv_values(dst->ne[1]);
- }
+ rctx.src0_nrows = kparams->src0_nrows;
+ rctx.src0_nrows_per_thread = kparams->src0_nrows_per_thread;
+ rctx.div_ne2_ne1 = kparams->div_ne2_ne1;
+ rctx.div_ne1 = kparams->div_ne1;
FARF(HIGH, "rope-f32 n-rows %u n-dims %d ne0 %u ext-factor %.6f theta-scale %.6f attn-factor %.6f\n", rctx.src0_nrows, rctx.n_dims, ne0,
rctx.ext_factor, rctx.theta_scale, rctx.attn_factor);
- if (!(octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)) {
- worker_pool_run_func(octx->ctx->worker_pool, rope_job_f32, &rctx, n_threads);
- }
+ work_queue_run(octx->ctx->work_queue, rope_job_f32, &rctx, kparams->n_threads);
return err;
}
diff --git a/ggml/src/ggml-hexagon/htp/rope-ops.h b/ggml/src/ggml-hexagon/htp/rope-ops.h
new file mode 100644
index 000000000..476653d05
--- /dev/null
+++ b/ggml/src/ggml-hexagon/htp/rope-ops.h
@@ -0,0 +1,56 @@
+#ifndef HTP_ROPE_OPS_H
+#define HTP_ROPE_OPS_H
+
+#include "hex-common.h"
+#include "hex-fastdiv.h"
+
+#define HTP_ROPE_SPAD_BLOCK 8
+#define HTP_ROPE_SPAD_NSLOTS 4
+#define HTP_ROPE_SPAD_NROWS (HTP_ROPE_SPAD_BLOCK * HTP_ROPE_SPAD_NSLOTS)
+
+struct htp_rope_kernel_params {
+ uint32_t n_threads;
+ uint32_t src0_nrows;
+ uint32_t src0_nrows_per_thread;
+ uint32_t vtcm_size;
+ uint32_t spad_per_thread;
+ uint32_t theta_cache_offset;
+ uint32_t src0_row_size_aligned;
+
+ struct fastdiv_values div_ne2_ne1;
+ struct fastdiv_values div_ne1;
+};
+
+#if defined(__cplusplus)
+static_assert(sizeof(struct htp_rope_kernel_params) <= 128, "htp_rope_kernel_params is too large for kernel_params blob");
+#else
+_Static_assert(sizeof(struct htp_rope_kernel_params) <= 128, "htp_rope_kernel_params is too large for kernel_params blob");
+#endif
+
+struct htp_rope_vtcm_layout {
+ size_t total_bytes;
+ size_t bytes_per_thread;
+ size_t theta_cache_size_aligned;
+ size_t src0_row_size_aligned;
+};
+
+static inline void htp_rope_vtcm_layout_build(
+ struct htp_rope_vtcm_layout * layout,
+ uint32_t ne00,
+ uint32_t n_threads
+) {
+ const size_t src0_row_size = ne00 * sizeof(float);
+ const size_t src0_row_size_aligned = hex_round_up((uint32_t) src0_row_size, 128);
+ const size_t theta_cache_size_aligned = hex_round_up((uint32_t) src0_row_size, 256);
+
+ layout->src0_row_size_aligned = src0_row_size_aligned;
+ layout->theta_cache_size_aligned = theta_cache_size_aligned;
+ layout->bytes_per_thread = theta_cache_size_aligned + HTP_ROPE_SPAD_NROWS * src0_row_size_aligned;
+ layout->total_bytes = layout->bytes_per_thread * n_threads;
+}
+
+static inline uint8_t * rope_spad_slot(uint8_t * base, uint32_t slot, size_t row_size_aligned) {
+ return base + (slot * HTP_ROPE_SPAD_BLOCK) * row_size_aligned;
+}
+
+#endif // HTP_ROPE_OPS_H
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index 8030186fb..2deb90f6a 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -10282,6 +10282,12 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_rope(type, {128, 32, 2, 1}, 32, GGML_ROPE_TYPE_NEOX, 512, 1.4245f, 0.7465f, 1.4245f, false, 0, true, true, 32));
}
+ // Real-model RoPE: F32 forward, packed Q, 512-token prefill.
+ test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {256, 8, 512, 1}, 64, GGML_ROPE_TYPE_IMROPE, 512, 1.0f, 0.0f, 1.0f, false, 0, true)); // qwen3.5 0.8B
+ test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {256, 16, 512, 1}, 64, GGML_ROPE_TYPE_IMROPE, 512, 1.0f, 0.0f, 1.0f, false, 0, true)); // qwen3.5 4B
+ test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {256, 8, 512, 1}, 256, GGML_ROPE_TYPE_NEOX, 512, 1.0f, 0.0f, 1.0f, false, 0, true)); // gemma4 E2B sliding
+ test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {512, 8, 512, 1}, 128, GGML_ROPE_TYPE_NEOX, 512, 1.0f, 0.0f, 1.0f, true, 0, true)); // gemma4 E4B global
+
for (int v : { 0, 1, 2, 3 }) {
for (int dim : { 0, 1, 2, 3, }) {
test_cases.emplace_back(new test_concat(GGML_TYPE_F32, {11, 12, 13, 14}, 7, dim, v));
@@ -11178,6 +11184,12 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() {
}
}
+ // Real-model RoPE: F32 forward, packed Q, 512-token prefill.
+ test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {256, 8, 512, 1}, 64, GGML_ROPE_TYPE_IMROPE, 512, 1.0f, 0.0f, 1.0f, false, 0, true)); // qwen3.5 0.8B
+ test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {256, 16, 512, 1}, 64, GGML_ROPE_TYPE_IMROPE, 512, 1.0f, 0.0f, 1.0f, false, 0, true)); // qwen3.5 4B
+ test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {256, 8, 512, 1}, 256, GGML_ROPE_TYPE_NEOX, 512, 1.0f, 0.0f, 1.0f, false, 0, true)); // gemma4 E2B sliding
+ test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {512, 8, 512, 1}, 128, GGML_ROPE_TYPE_NEOX, 512, 1.0f, 0.0f, 1.0f, true, 0, true)); // gemma4 E4B global
+
std::vector<std::array<int64_t, 4>> reduce_rows_cases = {
{ 8192, 1, 1, 1 },
{ 8192, 8192, 1, 1 },