Superminaren/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-04 02:37:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

context : autoscale n_ctx_train when yarn scaling specified (#28030)

Browse Source
This commit is contained in:
Sigbjørn Skjæret
2026-09-01 19:59:54 +03:00
committed by GitHub
parent f28493c783
commit dfc29b64eb
2 changed files with 16 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/llama-context.cpp
+15 -7
View File
@@ -125,8 +125,9 @@ llama_context::llama_context(
cparams.embeddings_layer_inp.resize(hparams.n_layer() + 1, false);
embd_layer_inp.resize(hparams.n_layer() + 1);
cparams.ctx_type = params.ctx_type;
cparams.pooling_type = params.pooling_type;
cparams.ctx_type = params.ctx_type;
cparams.rope_scaling_type = params.rope_scaling_type;
cparams.pooling_type = params.pooling_type;
cparams.n_ctx = params.n_ctx == 0 ? hparams.n_ctx_train : params.n_ctx;
cparams.rope_freq_base = params.rope_freq_base == 0.0f ? hparams.rope_freq_base_train : params.rope_freq_base;
@@ -160,17 +161,16 @@ llama_context::llama_context(
}
}
auto rope_scaling_type = params.rope_scaling_type;
if (rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED) {
rope_scaling_type = hparams.rope_scaling_type_train;
if (cparams.rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED) {
cparams.rope_scaling_type = hparams.rope_scaling_type_train;
}
if (rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_NONE) {
if (cparams.rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_NONE) {
cparams.rope_freq_scale = 1.0f; // never scale if scaling type is none
}
if (cparams.yarn_ext_factor < 0.0f) { // negative indicates 'not set'
cparams.yarn_ext_factor = rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_YARN ? 1.0f : 0.0f;
cparams.yarn_ext_factor = cparams.rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_YARN ? 1.0f : 0.0f;
}
if (cparams.yarn_ext_factor != 0) {
@@ -3734,6 +3734,14 @@ llama_context * llama_init_from_model(
try {
auto * ctx = new llama_context(*model, params);
const auto & cparams = ctx->get_cparams();
if (cparams.rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_YARN && cparams.rope_freq_scale != model->hparams.rope_freq_scale_train) {
LLAMA_LOG_INFO("%s: custom YaRN scaling detected, re-adjusting n_ctx_train(%u)...\n", __func__, model->hparams.n_ctx_train);
model->hparams.n_ctx_train = cparams.n_ctx_orig_yarn / cparams.rope_freq_scale;
LLAMA_LOG_INFO("%s: n_ctx_train adjusted to %u\n", __func__, model->hparams.n_ctx_train);
}
return ctx;
} catch (const std::exception & err) {
LLAMA_LOG_ERROR("%s: failed to initialize the context: %s\n", __func__, err.what());
src/llama-cparams.h
+1
View File
@@ -57,6 +57,7 @@ struct llama_cparams {
std::vector<bool> embeddings_layer_inp; // [n_layer()] extract input embeddings for layer
enum llama_context_type ctx_type;
enum llama_rope_scaling_type rope_scaling_type;
enum llama_pooling_type pooling_type;
ggml_backend_sched_eval_callback cb_eval;