From 4e275670e0a1beada06a2c8a96730304a7eee86f Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Tue, 1 Sep 2026 22:18:57 +0300 Subject: [PATCH] metal : add generic fusion stats via ad-hoc proc-address API Add a device-owned fusion context that lets a test tool count how many times each fusion pattern fires and toggle fusion. It is exposed through the ad-hoc ggml_backend_reg_get_proc_address mechanism with generic names so the testing tool is backend-agnostic: - ggml_backend_fusion_stats_init: start collecting fusion stats; when a context is created afterwards it registers the labels/counters and encodes single-threaded (n_cb == 0) so the counters are race-free - ggml_backend_fusion_stats_reset / _get_stats / _set_enabled The context lives on the metal device (not on the last backend context), so counters accumulate across contexts and reads are always consistent. The enable/disable toggle is initialized from GGML_METAL_FUSION_DISABLE and can be overridden by the test through set_enabled. Labels are synthesized from the fuse table via ggml_metal_fuse_label (e.g. "GATED_DELTA_NET+CPY"). Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 --- ggml/src/ggml-metal/ggml-metal-context.h | 2 + ggml/src/ggml-metal/ggml-metal-context.m | 69 ++++++++++++++++-------- ggml/src/ggml-metal/ggml-metal-device.h | 23 ++++++++ ggml/src/ggml-metal/ggml-metal-device.m | 57 ++++++++++++++++++++ ggml/src/ggml-metal/ggml-metal-fuse.cpp | 33 ++++++++++++ ggml/src/ggml-metal/ggml-metal-fuse.h | 36 ++++++++----- ggml/src/ggml-metal/ggml-metal-ops.cpp | 60 ++++++++++++++------- ggml/src/ggml-metal/ggml-metal-ops.h | 5 +- ggml/src/ggml-metal/ggml-metal.cpp | 32 +++++++++++ 9 files changed, 263 insertions(+), 54 deletions(-) diff --git a/ggml/src/ggml-metal/ggml-metal-context.h b/ggml/src/ggml-metal/ggml-metal-context.h index abf4b06ed2..f28a477a77 100644 --- a/ggml/src/ggml-metal/ggml-metal-context.h +++ b/ggml/src/ggml-metal/ggml-metal-context.h @@ -1,6 +1,7 @@ #pragma once #include "ggml-metal-device.h" +#include "ggml-metal-fuse.h" #ifdef __cplusplus extern "C" { @@ -33,6 +34,7 @@ ggml_metal_event_t ggml_metal_get_ev_cpy(ggml_metal_t ctx); void ggml_metal_set_n_cb (ggml_metal_t ctx, int n_cb); void ggml_metal_set_abort_callback (ggml_metal_t ctx, ggml_abort_callback abort_callback, void * user_data); + bool ggml_metal_supports_family (ggml_metal_t ctx, int family); void ggml_metal_capture_next_compute(ggml_metal_t ctx); diff --git a/ggml/src/ggml-metal/ggml-metal-context.m b/ggml/src/ggml-metal/ggml-metal-context.m index e1129db302..6e82d7d535 100644 --- a/ggml/src/ggml-metal/ggml-metal-context.m +++ b/ggml/src/ggml-metal/ggml-metal-context.m @@ -6,6 +6,7 @@ #import "ggml-metal-impl.h" #import "ggml-metal-common.h" #import "ggml-metal-ops.h" +#import "ggml-metal-fuse.h" #import @@ -36,15 +37,14 @@ struct ggml_metal { // additional, inference-time compiled pipelines ggml_metal_pipelines_t pipelines_ext; - bool use_fusion; bool use_concurrency; bool use_graph_optimize; int debug_graph; - int debug_fusion; - // how many times a given op was fused - uint64_t fuse_cnt[GGML_OP_COUNT]; + // shared fusion debugging context (NULL unless a test enabled fusion debugging) + // when non-NULL the graph is always encoded by a single thread (n_cb == 0) + struct ggml_metal_fusion * fusion; // capture state int capture_compute; @@ -138,7 +138,6 @@ ggml_metal_t ggml_metal_init(ggml_metal_device_t dev) { res->d_queue = dispatch_queue_create("ggml-metal", DISPATCH_QUEUE_CONCURRENT); - res->use_fusion = getenv("GGML_METAL_FUSION_DISABLE") == nil; res->use_concurrency = getenv("GGML_METAL_CONCURRENCY_DISABLE") == nil; { @@ -146,20 +145,35 @@ ggml_metal_t ggml_metal_init(ggml_metal_device_t dev) { res->debug_graph = val ? atoi(val) : 0; } - { - const char * val = getenv("GGML_METAL_FUSION_DEBUG"); - res->debug_fusion = val ? atoi(val) : 0; - } - res->use_graph_optimize = true; if (getenv("GGML_METAL_GRAPH_OPTIMIZE_DISABLE") != NULL) { res->use_graph_optimize = false; } - memset(res->fuse_cnt, 0, sizeof(res->fuse_cnt)); + // when fusion debugging is enabled the backend registers with the shared, device-owned + // fusion debugging context and forces single-threaded encoding (n_cb == 0) so the + // counters are race-free + res->fusion = ggml_metal_device_get_fusion(dev); + if (res->fusion->stats) { + if (!res->fusion->labels_set) { + int n = 0; + const ggml_metal_fuse * all = ggml_metal_fuse_all(&n); - GGML_LOG_INFO("%s: use fusion = %s\n", __func__, res->use_fusion ? "true" : "false"); + // labels point into the static fuse table; labels[]/counts[] are freed by the + // device when the debug context is destroyed + res->fusion->labels = calloc(n > 0 ? n : 1, sizeof(char *)); + res->fusion->counts = calloc(n > 0 ? n : 1, sizeof(uint64_t)); + for (int i = 0; i < n; i++) { + res->fusion->labels[i] = ggml_metal_fuse_label(&all[i]); + } + res->fusion->n_fusions = n; + res->fusion->labels_set = true; + } + res->n_cb = 0; + } + + GGML_LOG_INFO("%s: use fusion = %s\n", __func__, res->fusion->enabled ? "true" : "false"); GGML_LOG_INFO("%s: use concurrency = %s\n", __func__, res->use_concurrency ? "true" : "false"); GGML_LOG_INFO("%s: use graph optimize = %s\n", __func__, res->use_graph_optimize ? "true" : "false"); @@ -221,15 +235,16 @@ void ggml_metal_free(ggml_metal_t ctx) { ctx->pipelines_ext = nil; } - if (ctx->debug_fusion > 0) { + if (ctx->fusion->debug > 0) { GGML_LOG_DEBUG("%s: fusion stats:\n", __func__); - for (int i = 0; i < GGML_OP_COUNT; i++) { - if (ctx->fuse_cnt[i] == 0) { + + for (int i = 0; i < ctx->fusion->n_fusions; i++) { + if (ctx->fusion->counts[i] == 0) { continue; } // note: cannot use ggml_log here - GGML_LOG_DEBUG("%s: - %s: %" PRIu64 "\n", __func__, ggml_op_name((enum ggml_op) i), ctx->fuse_cnt[i]); + GGML_LOG_DEBUG("%s: - %s: %" PRIu64 "\n", __func__, ctx->fusion->labels[i], ctx->fusion->counts[i]); } } @@ -480,10 +495,17 @@ enum ggml_status ggml_metal_graph_compute(ggml_metal_t ctx, struct ggml_cgraph * @autoreleasepool { ctx->gf = gf; - ctx->n_nodes_0 = MIN(n_main, gf->n_nodes); - ctx->n_nodes_1 = gf->n_nodes - ctx->n_nodes_0; + if (ctx->n_cb == 0) { + // single-threaded encoding: the whole graph is encoded by one command buffer + ctx->n_nodes_0 = gf->n_nodes; + ctx->n_nodes_1 = 0; + ctx->n_nodes_per_cb = 0; + } else { + ctx->n_nodes_0 = MIN(n_main, gf->n_nodes); + ctx->n_nodes_1 = gf->n_nodes - ctx->n_nodes_0; - ctx->n_nodes_per_cb = (ctx->n_nodes_1 + ctx->n_cb - 1) / ctx->n_cb; + ctx->n_nodes_per_cb = (ctx->n_nodes_1 + ctx->n_cb - 1) / ctx->n_cb; + } if (ctx->capture_compute >= 0) { ctx->capture_compute--; @@ -681,6 +703,12 @@ ggml_metal_event_t ggml_metal_get_ev_cpy(ggml_metal_t ctx) { } void ggml_metal_set_n_cb(ggml_metal_t ctx, int n_cb) { + // when fusion stats are collected the graph must be encoded by a single thread so the + // counters are race-free; override whatever the caller requested + if (ctx->fusion->stats) { + n_cb = 0; + } + if (ctx->n_cb != n_cb) { ctx->n_cb = MIN(n_cb, GGML_METAL_MAX_COMMAND_BUFFERS); @@ -718,11 +746,10 @@ void ggml_metal_set_n_cb(ggml_metal_t ctx, int n_cb) { ctx->gf, idx_start, idx_end, - ctx->use_fusion, ctx->use_concurrency, ctx->capture_compute, ctx->debug_graph, - ctx->debug_fusion); + ctx->fusion); for (int idx = 0; idx < ggml_metal_op_n_nodes(ctx_op); ++idx) { const int res = ggml_metal_op_encode(ctx_op, idx); diff --git a/ggml/src/ggml-metal/ggml-metal-device.h b/ggml/src/ggml-metal/ggml-metal-device.h index 31fc07d44d..bce0305b5a 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.h +++ b/ggml/src/ggml-metal/ggml-metal-device.h @@ -325,6 +325,29 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te const struct ggml_metal_device_props * ggml_metal_device_get_props(ggml_metal_device_t dev); +// shared fusion debugging context, owned by the device; newly created backend contexts for that +// device register with it so the fusion counters are race-free and accumulate across contexts. +// labels[] points into the static fuse table; labels[]/counts[] are allocated with calloc and +// freed when the context is destroyed. +struct ggml_metal_fusion { + int n_fusions; // number of fusion patterns + const char ** labels; // one label per pattern (set by the context) + uint64_t * counts; // one counter per pattern (incremented by the op encoders) + bool enabled; // whether the backend actually fuses (set by the test) + bool stats; // whether to collect fusion stats + bool labels_set; // whether the labels have been registered yet + int debug; // env: GGML_METAL_FUSION_DEBUG +}; + +// the device-owned fusion debugging context (NULL unless fusion debugging is enabled) +struct ggml_metal_fusion * ggml_metal_device_get_fusion(ggml_metal_device_t dev); + +// fusion debugging control (backend side of the ad-hoc ggml_backend_debug_fusion_* proc-address API) +void ggml_metal_device_fusion_stats_init (ggml_metal_device_t dev); +void ggml_metal_device_fusion_stats_reset(ggml_metal_device_t dev); +int ggml_metal_device_fusion_stats_get (ggml_metal_device_t dev, const char ** labels, uint64_t * counts, int n); +void ggml_metal_device_fusion_set_enabled(ggml_metal_device_t dev, bool enabled); + // // device buffers // diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m index e20a4e8916..3a7b2b3a86 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.m +++ b/ggml/src/ggml-metal/ggml-metal-device.m @@ -898,6 +898,9 @@ struct ggml_metal_device { // virtual address for GPU memory allocations atomic_uintptr_t addr_virt; + + // shared fusion debugging context (NULL unless a test enabled fusion debugging) + struct ggml_metal_fusion * fusion; }; // @@ -1274,6 +1277,17 @@ ggml_metal_device_t ggml_metal_device_init(int device, int n_devices) { dev->props.max_working_set_size = dev->mtl_device.maxBufferLength; } + dev->fusion = calloc(1, sizeof(struct ggml_metal_fusion)); + dev->fusion->enabled = getenv("GGML_METAL_FUSION_DISABLE") == nil; + + { + const char * val = getenv("GGML_METAL_FUSION_DEBUG"); + dev->fusion->debug = val ? atoi(val) : 0; + if (dev->fusion->debug > 0) { + dev->fusion->stats = true; + } + } + snprintf(dev->props.name, sizeof(dev->props.name), "%s%d", "MTL", device); const char * gpu_name = [[dev->mtl_device name] UTF8String]; if (n_devices > 1) { @@ -1348,6 +1362,10 @@ void ggml_metal_device_free(ggml_metal_device_t dev) { assert(dev != NULL); @autoreleasepool { + free(dev->fusion->labels); + free(dev->fusion->counts); + free(dev->fusion); + ggml_metal_rsets_free(dev->rsets); ggml_metal_library_free(dev->library); @@ -1932,6 +1950,45 @@ static void ggml_metal_device_disable_tensor(ggml_metal_device_t dev) { dev->props.has_tensor = false; } +struct ggml_metal_fusion * ggml_metal_device_get_fusion(ggml_metal_device_t dev) { + return dev->fusion; +} + +void ggml_metal_device_fusion_stats_init(ggml_metal_device_t dev) { + dev->fusion->stats = true; +} + +void ggml_metal_device_fusion_stats_reset(ggml_metal_device_t dev) { + if (dev->fusion != NULL && dev->fusion->counts != NULL) { + memset(dev->fusion->counts, 0, dev->fusion->n_fusions * sizeof(uint64_t)); + } +} + +int ggml_metal_device_fusion_stats_get(ggml_metal_device_t dev, const char ** labels, uint64_t * counts, int n) { + if (dev->fusion == NULL) { + return 0; + } + + // query: report how many fusion patterns are available + if (labels == NULL) { + return dev->fusion->n_fusions; + } + + const int n_fill = MIN(n, dev->fusion->n_fusions); + for (int i = 0; i < n_fill; i++) { + labels[i] = dev->fusion->labels[i]; + if (counts != NULL) { + counts[i] = dev->fusion->counts[i]; + } + } + + return n_fill; +} + +void ggml_metal_device_fusion_set_enabled(ggml_metal_device_t dev, bool enabled) { + dev->fusion->enabled = enabled; +} + // // device buffers // diff --git a/ggml/src/ggml-metal/ggml-metal-fuse.cpp b/ggml/src/ggml-metal/ggml-metal-fuse.cpp index 93fcb50f95..d8d0f3941b 100644 --- a/ggml/src/ggml-metal/ggml-metal-fuse.cpp +++ b/ggml/src/ggml-metal/ggml-metal-fuse.cpp @@ -4,6 +4,7 @@ #include "ggml-metal-device.h" #include +#include // ---- helpers ------------------------------------------------------------- @@ -236,6 +237,38 @@ const ggml_metal_fuse * ggml_metal_fuse_all(int * n) { return ggml_metal_fuses; } +const char * ggml_metal_fuse_label(const ggml_metal_fuse * fuse) { + const int n_fusions = (int) sizeof(ggml_metal_fuses) / sizeof(ggml_metal_fuses[0]); + + const int idx = (int)(fuse - ggml_metal_fuses); + GGML_ASSERT(idx >= 0 && idx < n_fusions); + + // labels are built once and cached (the table is static, so the pointers stay valid) + static char labels[sizeof(ggml_metal_fuses) / sizeof(ggml_metal_fuses[0])][GGML_METAL_FUSE_LABEL_MAX]; + static bool built = false; + + if (!built) { + for (int i = 0; i < n_fusions; i++) { + char * buf = labels[i]; + int len = 0; + for (int j = 0; j < ggml_metal_fuses[i].n_ops; j++) { + if (j > 0) { + buf[len++] = '+'; + } + const char * name = ggml_op_name(ggml_metal_fuses[i].ops[j]); + const int name_len = (int) strlen(name); + GGML_ASSERT(len + name_len < GGML_METAL_FUSE_LABEL_MAX); + memcpy(buf + len, name, name_len); + len += name_len; + } + buf[len] = '\0'; + } + built = true; + } + + return labels[idx]; +} + // ---- queries ------------------------------------------------------------- // find the longest pattern matching the node sequence starting at idx diff --git a/ggml/src/ggml-metal/ggml-metal-fuse.h b/ggml/src/ggml-metal/ggml-metal-fuse.h index edf6d0f87d..a81c4171f2 100644 --- a/ggml/src/ggml-metal/ggml-metal-fuse.h +++ b/ggml/src/ggml-metal/ggml-metal-fuse.h @@ -19,30 +19,30 @@ extern "C" { // (also the maximum length of a packed fusion group during graph optimization) #define GGML_METAL_FUSE_MAX 16 -enum ggml_metal_fuse_mode { +typedef enum ggml_metal_fuse_mode { // structural checks only; used by the graph optimizer, at which point the graph // tensors are not allocated yet, so buffer placement cannot be verified GGML_METAL_FUSE_STRUCTURAL = 0, // full checks, including buffer placement; used by the op encoders GGML_METAL_FUSE_FULL, -}; +} ggml_metal_fuse_mode; // identifier of each fusion pattern so the op encoders know which kernel to use -enum ggml_metal_fuse_id { +typedef enum ggml_metal_fuse_id { GGML_METAL_FUSE_NONE = 0, GGML_METAL_FUSE_NORM_MUL, // NORM/RMS_NORM + MUL GGML_METAL_FUSE_NORM_MUL_ADD, // NORM/RMS_NORM + MUL + ADD GGML_METAL_FUSE_ADD_CHAIN, // ADD x N (N in [2, 7]) GGML_METAL_FUSE_SNAKE, // MUL + SIN + SQR + MUL + ADD GGML_METAL_FUSE_GDN_CACHE, // GATED_DELTA_NET + CPY (write snapshots into the recurrent cache) -}; +} ggml_metal_fuse_id; struct ggml_metal_fuse { ggml_metal_fuse_id id; - const ggml_op * ops; // op sequence (fixed length) - int n_ops; // number of ops - const int * outputs; // output node indices (absolute graph indices; nullptr => the last node) - int n_outputs; // number of outputs (0 => default last node) + const enum ggml_op * ops; // op sequence (fixed length) + int n_ops; // number of ops + const int * outputs; // output node indices (absolute graph indices; nullptr => the last node) + int n_outputs; // number of outputs (0 => default last node) // if unsafe: the generic chain/shape + ggml_can_fuse_subgraph checks are skipped and the // check callback below is the sole validator (used for patterns that are not elision chains, @@ -51,18 +51,28 @@ struct ggml_metal_fuse { // extra backend constraints on top of ggml_can_fuse_subgraph // nodes[j] is the j-th node of the pattern - bool (*check)(const ggml_tensor * const * nodes, - const ggml_metal_fuse * fuse, - ggml_metal_fuse_mode mode); + bool (*check)(const struct ggml_tensor * const * nodes, + const struct ggml_metal_fuse * fuse, + ggml_metal_fuse_mode mode); }; +typedef struct ggml_metal_fuse ggml_metal_fuse; + // the single table of all fusions supported by the Metal backend const ggml_metal_fuse * ggml_metal_fuse_all(int * n); +// stable, human-readable label of a fuse entry describing the exact op sequence, +// e.g. "NORM+MUL", "ADD+ADD+ADD", "GATED_DELTA_NET+CPY". the returned pointer stays +// valid for the lifetime of the process (the fuse table is static). +const char * ggml_metal_fuse_label(const ggml_metal_fuse * fuse); + +// maximum label length (op names + separators) +#define GGML_METAL_FUSE_LABEL_MAX 64 + // compute phase: longest fusion starting at idx (a position in node_idxs) that matches in `mode`. // returns the matching pattern (nullptr if no fusion) and sets *n_out to the number of nodes consumed. const ggml_metal_fuse * ggml_metal_fuse_next( - const ggml_cgraph * gf, + const struct ggml_cgraph * gf, const int * node_idxs, int n_idxs, int idx, @@ -71,7 +81,7 @@ const ggml_metal_fuse * ggml_metal_fuse_next( // optimize phase: maximum number of nodes starting at idx (a raw sequential graph index) that // could be fused, chaining patterns back-to-back. returns at least 1. -int ggml_metal_fuse_max(const ggml_cgraph * gf, int idx); +int ggml_metal_fuse_max(const struct ggml_cgraph * gf, int idx); #ifdef __cplusplus } diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp index bd93175fd0..986a7011e5 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.cpp +++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp @@ -34,22 +34,20 @@ struct ggml_metal_op { ggml_cgraph * gf, int idx_start, int idx_end, - bool use_fusion, bool use_concurrency, bool use_capture, int debug_graph, - int debug_fusion) { + ggml_metal_fusion * fusion) { this->dev = dev; this->lib = ggml_metal_device_get_library(dev); this->enc = ggml_metal_encoder_init(cmd_buf, use_concurrency); this->mem_ranges = ggml_mem_ranges_init(debug_graph); this->idx_start = idx_start; this->idx_end = idx_end; - this->use_fusion = use_fusion; this->use_concurrency = use_concurrency; this->use_capture = use_capture; this->debug_graph = debug_graph; - this->debug_fusion = debug_fusion; + this->fusion = fusion; this->gf = gf; idxs.reserve(gf->n_nodes); @@ -82,23 +80,44 @@ struct ggml_metal_op { // consult the fusion table for the longest pattern starting at i0 // returns the matching pattern (nullptr if no fusion) and sets *n_out to the number of nodes const ggml_metal_fuse * can_fuse(int i0, enum ggml_metal_fuse_mode mode, int * n_out) const { - assert(use_fusion); + assert(use_fusion()); assert(i0 >= 0 && i0 < n_nodes()); return ggml_metal_fuse_next(gf, idxs.data(), (int) idxs.size(), i0, mode, n_out); } + // whether to attempt fusion; the toggle lives in the shared fusion debugging context owned + // by the device (initialized from GGML_METAL_FUSION_DISABLE, overridable by the test) + bool use_fusion() const { + return fusion->enabled; + } + + // record that a fusion fired, indexed by the matching table entry + void count_fuse(const ggml_metal_fuse * fuse) const { + if (!fusion->stats || fuse == nullptr) { + return; + } + + int n = 0; + const ggml_metal_fuse * all = ggml_metal_fuse_all(&n); + const int idx = (int)(fuse - all); + if (idx >= 0 && idx < n) { + fusion->counts[idx]++; + } + } + ggml_metal_device_t dev; ggml_metal_library_t lib; ggml_metal_encoder_t enc; ggml_mem_ranges_t mem_ranges; - bool use_fusion; bool use_concurrency; bool use_capture; int debug_graph; - int debug_fusion; + + // shared fusion debugging context (NULL unless fusion debugging is enabled) + struct ggml_metal_fusion * fusion; private: ggml_cgraph * gf; @@ -116,22 +135,20 @@ ggml_metal_op_t ggml_metal_op_init( ggml_cgraph * gf, int idx_start, int idx_end, - bool use_fusion, bool use_concurrency, bool use_capture, int debug_graph, - int debug_fusion) { + struct ggml_metal_fusion * fusion) { ggml_metal_op_t res = new ggml_metal_op( dev, cmd_buf, gf, idx_start, idx_end, - use_fusion, use_concurrency, use_capture, debug_graph, - debug_fusion); + fusion); return res; } @@ -1867,8 +1884,8 @@ int ggml_metal_op_gated_delta_net(ggml_metal_op_t ctx, int idx) { ggml_metal_library_t lib = ctx->lib; ggml_metal_encoder_t enc = ctx->enc; - const bool use_fusion = ctx->use_fusion; - const int debug_fusion = ctx->debug_fusion; + const bool use_fusion = ctx->use_fusion(); + const int debug_fusion = ctx->fusion->debug; GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); @@ -1898,6 +1915,8 @@ int ggml_metal_op_gated_delta_net(ggml_metal_op_t ctx, int idx) { nb_out = dst_cache->nb[2]/sizeof(float); n_fuse = 2; + ctx->count_fuse(fuse); + if (debug_fusion > 1) { GGML_LOG_DEBUG("%s: fuse: GATED_DELTA_NET + CPY\n", __func__); } @@ -3748,13 +3767,14 @@ int ggml_metal_op_bin(ggml_metal_op_t ctx, int idx) { int n_fuse = 1; const ggml_metal_fuse * fuse = nullptr; - if (ctx->use_fusion) { + if (ctx->use_fusion()) { int n = 1; fuse = ctx->can_fuse(idx, GGML_METAL_FUSE_FULL, &n); n_fuse = n; // snake activation autofuse: mul -> sin -> sqr -> mul -> add if (fuse && fuse->id == GGML_METAL_FUSE_SNAKE) { + ctx->count_fuse(fuse); return ggml_metal_op_snake_fused(ctx, idx); } } @@ -3764,9 +3784,9 @@ int ggml_metal_op_bin(ggml_metal_op_t ctx, int idx) { ggml_metal_library_t lib = ctx->lib; ggml_metal_encoder_t enc = ctx->enc; - const bool use_fusion = ctx->use_fusion; + const bool use_fusion = ctx->use_fusion(); - const int debug_fusion = ctx->debug_fusion; + const int debug_fusion = ctx->fusion->debug; GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); @@ -3821,6 +3841,8 @@ int ggml_metal_op_bin(ggml_metal_op_t ctx, int idx) { args.o1[i] = ggml_metal_get_buffer_id(ctx->node(idx + i)->src[1]).offs; } + ctx->count_fuse(fuse); + if (debug_fusion > 1) { GGML_LOG_DEBUG("%s: fuse: ADD x %d\n", __func__, n_fuse); } @@ -4029,9 +4051,9 @@ int ggml_metal_op_norm(ggml_metal_op_t ctx, int idx) { ggml_metal_library_t lib = ctx->lib; ggml_metal_encoder_t enc = ctx->enc; - const bool use_fusion = ctx->use_fusion; + const bool use_fusion = ctx->use_fusion(); - const int debug_fusion = ctx->debug_fusion; + const int debug_fusion = ctx->fusion->debug; GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); @@ -4073,6 +4095,8 @@ int ggml_metal_op_norm(ggml_metal_op_t ctx, int idx) { if (fuse && (fuse->id == GGML_METAL_FUSE_NORM_MUL || fuse->id == GGML_METAL_FUSE_NORM_MUL_ADD)) { n_fuse = n; + ctx->count_fuse(fuse); + for (int i = 1; i < n_fuse; i++) { const ggml_tensor * fn = ctx->node(idx + i); diff --git a/ggml/src/ggml-metal/ggml-metal-ops.h b/ggml/src/ggml-metal/ggml-metal-ops.h index f8fe50b468..c73ae44356 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.h +++ b/ggml/src/ggml-metal/ggml-metal-ops.h @@ -8,17 +8,18 @@ extern "C" { typedef struct ggml_metal_op * ggml_metal_op_t; +struct ggml_metal_fusion; // forward decl (ggml-metal-device.h) + ggml_metal_op_t ggml_metal_op_init( ggml_metal_device_t dev, ggml_metal_cmd_buf_t cmd_buf, struct ggml_cgraph * gf, int idx_start, int idx_end, - bool use_fusion, bool use_concurrency, bool use_capture, int debug_graph, - int debug_fusion); + struct ggml_metal_fusion * fusion); void ggml_metal_op_free(ggml_metal_op_t ctx); diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal.cpp index 3bd6abd06f..621f8eb548 100644 --- a/ggml/src/ggml-metal/ggml-metal.cpp +++ b/ggml/src/ggml-metal/ggml-metal.cpp @@ -906,6 +906,24 @@ static const char * ggml_backend_metal_tuning_device_token(ggml_backend_dev_t de return ggml_metal_device_id_token(ggml_metal_device_get_props(ctx_dev)->device_id); } +// generic fusion debugging API (ad-hoc proc-address mechanism): these operate on the device and +// reach the shared fusion debugging context owned by the device +static void ggml_backend_metal_fusion_stats_init(ggml_backend_dev_t dev) { + ggml_metal_device_fusion_stats_init((ggml_metal_device_t)dev->context); +} + +static void ggml_backend_metal_fusion_stats_reset(ggml_backend_dev_t dev) { + ggml_metal_device_fusion_stats_reset((ggml_metal_device_t)dev->context); +} + +static int ggml_backend_metal_fusion_stats_get(ggml_backend_dev_t dev, const char ** labels, uint64_t * counts, int n) { + return ggml_metal_device_fusion_stats_get((ggml_metal_device_t)dev->context, labels, counts, n); +} + +static void ggml_backend_metal_fusion_set_enabled(ggml_backend_dev_t dev, bool enabled) { + ggml_metal_device_fusion_set_enabled((ggml_metal_device_t)dev->context, enabled); +} + static void * ggml_backend_metal_get_proc_address(ggml_backend_reg_t reg, const char * name) { if (strcmp(name, "ggml_backend_get_features") == 0) { return (void *)ggml_backend_metal_get_features; @@ -928,6 +946,20 @@ static void * ggml_backend_metal_get_proc_address(ggml_backend_reg_t reg, const if (strcmp(name, "ggml_backend_metal_tuning_device_token") == 0) { return (void *)ggml_backend_metal_tuning_device_token; } + // generic fusion debugging API (ad-hoc proc-address mechanism, not part of the official + // ggml backend interface yet; a backend that adopts it exports these exact names) + if (strcmp(name, "ggml_backend_fusion_stats_init") == 0) { + return (void *)ggml_backend_metal_fusion_stats_init; + } + if (strcmp(name, "ggml_backend_fusion_stats_reset") == 0) { + return (void *)ggml_backend_metal_fusion_stats_reset; + } + if (strcmp(name, "ggml_backend_fusion_stats_get") == 0) { + return (void *)ggml_backend_metal_fusion_stats_get; + } + if (strcmp(name, "ggml_backend_fusion_set_enabled") == 0) { + return (void *)ggml_backend_metal_fusion_set_enabled; + } return NULL;