From 851cb34f213317d799f31976df5dad95af7b41d3 Mon Sep 17 00:00:00 2001 From: Aparna M P Date: Sat, 19 Sep 2026 22:18:07 +0530 Subject: [PATCH] hexagon: add support for GEGLU_QUICK (#29114) --- ggml/src/ggml-hexagon/ggml-hexagon.cpp | 2 + ggml/src/ggml-hexagon/htp/act-ops.c | 68 +++++++++++++++++++++++++- ggml/src/ggml-hexagon/htp/htp-ops.h | 1 + ggml/src/ggml-hexagon/htp/main.c | 1 + 4 files changed, 71 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp index a7d7f38076..f84764536b 100644 --- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp +++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp @@ -5749,6 +5749,7 @@ static htp_op_code op_remap_to_htp(const ggml_tensor * t) { case GGML_GLU_OP_SWIGLU_OAI: return HTP_OP_GLU_SWIGLU_OAI; case GGML_GLU_OP_SWIGLU_CLAMP: return HTP_OP_GLU_SWIGLU_CLAMP; case GGML_GLU_OP_GEGLU: return HTP_OP_GLU_GEGLU; + case GGML_GLU_OP_GEGLU_QUICK: return HTP_OP_GLU_GEGLU_QUICK; default: break; } break; @@ -6769,6 +6770,7 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons case GGML_GLU_OP_SWIGLU_OAI: case GGML_GLU_OP_SWIGLU_CLAMP: case GGML_GLU_OP_GEGLU: + case GGML_GLU_OP_GEGLU_QUICK: supp = ggml_hexagon_supported_activations(sess, op); break; default: diff --git a/ggml/src/ggml-hexagon/htp/act-ops.c b/ggml/src/ggml-hexagon/htp/act-ops.c index 5fff372f28..5911c08900 100644 --- a/ggml/src/ggml-hexagon/htp/act-ops.c +++ b/ggml/src/ggml-hexagon/htp/act-ops.c @@ -312,6 +312,45 @@ static inline void hvx_geglu_f32_aa(uint8_t * restrict dst, const uint8_t * rest } } +static inline void hvx_geglu_quick_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src0 % 128 == 0); + assert((unsigned long) src1 % 128 == 0); + + HVX_Vector * restrict vdst = (HVX_Vector *) dst; + const HVX_Vector * restrict vsrc0 = (const HVX_Vector *) src0; + const HVX_Vector * restrict vsrc1 = (const HVX_Vector *) src1; + + const uint32_t epv = 128 / sizeof(float); + const uint32_t nvec = n / epv; + const uint32_t nloe = n % epv; + + const HVX_Vector v_scale = hvx_vec_splat_f32(1.702f); + const HVX_Vector v_one = hvx_vec_splat_f32(1.0f); + const HVX_Vector v_max_exp = hvx_vec_splat_f32(87.0f); + const HVX_Vector v_min_exp = hvx_vec_splat_f32(-87.0f); + + uint32_t i = 0; + + _Pragma("unroll(4)") + for (; i < nvec; i++) { + HVX_Vector x = vsrc0[i]; + HVX_Vector g = vsrc1[i]; + HVX_Vector scaled_x = hvx_vec_mul_f32_f32(x, v_scale); + HVX_Vector sigmoid_x = hvx_vec_fast_sigmoid_f32_guard_2it(scaled_x, v_one, v_max_exp, v_min_exp); + vdst[i] = hvx_vec_mul_f32_f32(hvx_vec_mul_f32_f32(x, sigmoid_x), g); + } + + if (nloe) { + HVX_Vector x = vsrc0[i]; + HVX_Vector g = vsrc1[i]; + HVX_Vector scaled_x = hvx_vec_mul_f32_f32(x, v_scale); + HVX_Vector sigmoid_x = hvx_vec_fast_sigmoid_f32_guard_2it(scaled_x, v_one, v_max_exp, v_min_exp); + HVX_Vector result = hvx_vec_mul_f32_f32(hvx_vec_mul_f32_f32(x, sigmoid_x), g); + hvx_vec_store_a((void *) &vdst[i], nloe * sizeof(float), result); + } +} + // geglu(x, g) = gelu(x) * g static void geglu_f32(const float * restrict src0, const float * restrict src1, @@ -329,6 +368,23 @@ static void geglu_f32(const float * restrict src0, } } +// geglu_quick(x, g) = x * sigmoid(1.702 * x) * g +static void geglu_quick_f32(const float * restrict src0, + const float * restrict src1, + float * restrict dst, + const uint32_t num_rows, + const struct htp_act_context * actx) { + htp_glu_op_preamble; + + for (uint32_t ib = 0; ib < num_rows; ib++) { + const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned); + const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned); + uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned); + + hvx_geglu_quick_f32_aa(dst_ptr, src0_ptr, src1_ptr, nc); + } +} + #define DEFINE_GLU_PER_THREAD(NAME, OP_STR, CORE_EXPR) \ static void glu_##NAME##_f32_per_thread(unsigned int nth, unsigned int ith, void * data) { \ struct htp_act_context * actx = (struct htp_act_context *) data; \ @@ -433,6 +489,7 @@ DEFINE_GLU_PER_THREAD(swiglu, "swiglu-f32", swiglu_f32(src0_spad, src1_spad, dst DEFINE_GLU_PER_THREAD(swiglu_oai, "swiglu-oai-f32", swiglu_oai_f32(src0_spad, src1_spad, dst_spad, block_size, actx)) DEFINE_GLU_PER_THREAD(swiglu_clamp, "swiglu-clamp-f32", swiglu_clamp_f32(src0_spad, src1_spad, dst_spad, block_size, actx)) DEFINE_GLU_PER_THREAD(geglu, "geglu-f32", geglu_f32(src0_spad, src1_spad, dst_spad, block_size, actx)) +DEFINE_GLU_PER_THREAD(geglu_quick, "geglu-quick-f32", geglu_quick_f32(src0_spad, src1_spad, dst_spad, block_size, actx)) static int execute_op_activations_f32(struct htp_ops_context * octx) { const struct htp_tensor * src0 = octx->src[0]; @@ -467,6 +524,11 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) { act_op_func = (worker_callback_t)glu_geglu_f32_per_thread; op_type = "geglu-f32"; break; + + case HTP_OP_GLU_GEGLU_QUICK: + act_op_func = (worker_callback_t)glu_geglu_quick_f32_per_thread; + op_type = "geglu-quick-f32"; + break; default: FARF(ERROR, "Unsupported activations Op %u\n", octx->op); return HTP_STATUS_NO_SUPPORT; @@ -570,7 +632,11 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) { const uint8_t * data_src0 = (const uint8_t *) src0->data; const uint8_t * data_src1 = src1 ? (const uint8_t *) src1->data : NULL; - if (!src1 && (octx->op == HTP_OP_GLU_SWIGLU || octx->op == HTP_OP_GLU_SWIGLU_OAI || octx->op == HTP_OP_GLU_SWIGLU_CLAMP || octx->op == HTP_OP_GLU_GEGLU)) { + if (!src1 && (octx->op == HTP_OP_GLU_SWIGLU || + octx->op == HTP_OP_GLU_SWIGLU_OAI || + octx->op == HTP_OP_GLU_SWIGLU_CLAMP || + octx->op == HTP_OP_GLU_GEGLU || + octx->op == HTP_OP_GLU_GEGLU_QUICK)) { const int32_t swapped = octx->op_params[1]; data_src1 = data_src0; actx.src1_row_size = actx.src0_row_size; diff --git a/ggml/src/ggml-hexagon/htp/htp-ops.h b/ggml/src/ggml-hexagon/htp/htp-ops.h index f978517da7..faf3118c49 100644 --- a/ggml/src/ggml-hexagon/htp/htp-ops.h +++ b/ggml/src/ggml-hexagon/htp/htp-ops.h @@ -71,6 +71,7 @@ enum htp_op_code { HTP_OP_GLU_SWIGLU, HTP_OP_GLU_SWIGLU_OAI, HTP_OP_GLU_GEGLU, + HTP_OP_GLU_GEGLU_QUICK, HTP_OP_SOFTMAX, HTP_OP_ADD_ID, HTP_OP_ROPE, diff --git a/ggml/src/ggml-hexagon/htp/main.c b/ggml/src/ggml-hexagon/htp/main.c index 16b6c14187..b324cfd3a3 100644 --- a/ggml/src/ggml-hexagon/htp/main.c +++ b/ggml/src/ggml-hexagon/htp/main.c @@ -828,6 +828,7 @@ static int execute_op(struct htp_ops_context * octx) { case HTP_OP_GLU_SWIGLU_OAI: case HTP_OP_GLU_SWIGLU_CLAMP: case HTP_OP_GLU_GEGLU: + case HTP_OP_GLU_GEGLU_QUICK: return op_activations(octx); case HTP_OP_SOFTMAX: