Superminaren/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-04 02:37:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

finetune: fix no KV cache (#27199)

Browse Source 
* training: fix no KV cache

* apply @ ggerganov
 suggestion
This commit is contained in:
Xuan-Son Nguyen
2026-09-02 23:53:32 +02:00
committed by GitHub
parent 9cffdcc801
commit 159b741427
3 changed files with 14 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/training/README.md
+2
View File
@@ -6,6 +6,8 @@ Finetuning of Stories 260K and LLaMA 3.2 1b seems to work with 24 GB of memory.
**For CPU training, compile llama.cpp without any additional backends such as CUDA.**
**For CUDA training, use the maximum number of GPU layers.**
Flash attention is disabled during training because `FLASH_ATTN_EXT` has no backward pass.
Proof of concept:
``` sh
ggml/src/ggml.c
+1 -1
View File
@@ -7335,7 +7335,7 @@ void ggml_build_backward_expand(
}
// inplace operations are currently not supported
GGML_ASSERT(!node->view_src || node->op == GGML_OP_CPY || node->op == GGML_OP_VIEW ||
GGML_ASSERT(!node->view_src || node->op == GGML_OP_CPY || node->op == GGML_OP_SET_ROWS || node->op == GGML_OP_VIEW ||
node->op == GGML_OP_RESHAPE || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_TRANSPOSE);
const size_t ihash = ggml_hash_find(&cgraph->visited_hash_set, node);
src/llama-context.cpp
+11 -1
View File
@@ -482,7 +482,8 @@ llama_context::~llama_context() {
// wait for any pending asynchronous copies into the output buffers before they are freed
synchronize();
if (!model.hparams.no_alloc) {
// when training, ggml_opt allocates extra buffers through the scheduler, so the sizes no longer match the expectation
if (!model.hparams.no_alloc && !opt_ctx) {
for (size_t i = 0; i < backend_ptrs.size(); ++i) {
ggml_backend_t backend = backend_ptrs[i];
ggml_backend_buffer_type_t buft = backend_buft[i];
@@ -3408,6 +3409,15 @@ void llama_context::opt_init(struct llama_model * model, struct llama_opt_params
GGML_ASSERT(model->hparams.n_ctx_train % n_batch == 0);
GGML_ASSERT(n_batch % n_ubatch == 0);
if (cparams.flash_attn) {
LLAMA_LOG_INFO("%s: disabling flash attention, FLASH_ATTN_EXT has no backward pass\n", __func__);
cparams.flash_attn = false;
// the graph changes without flash attention, need to reserve again
sched_need_reserve = true;
sched_reserve();
}
ggml_opt_params opt_params = ggml_opt_default_params(sched.get(), GGML_OPT_LOSS_TYPE_CROSS_ENTROPY);
opt_params.opt_period = n_batch / n_ubatch;
opt_params.get_opt_pars = lopt_params.get_opt_pars;