From 106369ac16318ba818f244d22d66a7ddfda76a5b Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Tue, 3 Nov 2026 19:43:21 -0600 Subject: [PATCH 1/4] --checkpoint-min-step auto, and an uncapped request decodes at most its slot's window: the engine derives both from the window it holds Continuum launches with -c unset and --fit on (continuum card 4a20255d), so the slot window is the engine's, known only after the fit. Two launch flags used to restate a window the caller guessed: - --checkpoint-min-step auto: the spacing is ceil(slot window / --ctx-checkpoints), computed once the slot window is known, so the checkpoints cover a whole slot (continuum's checkpoint_min_step rule); - a request with no n_predict decodes at most its slot's window instead of forever (continuum card 912e606d), so the caller no longer passes -n. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- common/arg.cpp | 17 +++++++++++------ tools/server/server-context.cpp | 16 +++++++++++++++- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/common/arg.cpp b/common/arg.cpp index c3b540f5a5fb..7148cf5af519 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -1692,13 +1692,18 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } ).set_env("LLAMA_ARG_CTX_CHECKPOINTS").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); add_opt(common_arg( - {"-cms", "--checkpoint-min-step"}, "N", - string_format("minimum spacing between context checkpoints in tokens (default: %d, 0 = no minimum)", params.checkpoint_min_step), - [](common_params & params, int value) { - if (value < 0) { - throw std::invalid_argument("checkpoint-min-step must be non-negative"); + {"-cms", "--checkpoint-min-step"}, "N|auto", + string_format("minimum spacing between context checkpoints in tokens (default: %d, 0 = no minimum, auto = the slot's window over --ctx-checkpoints, so the checkpoints cover a whole slot)", params.checkpoint_min_step), + [](common_params & params, const std::string & value) { + if (value == "auto") { + params.checkpoint_min_step = -1; // derived from the engine's own slot window once it is known + return; + } + const int n = std::stoi(value); + if (n < 0) { + throw std::invalid_argument("checkpoint-min-step must be non-negative or auto"); } - params.checkpoint_min_step = value; + params.checkpoint_min_step = n; } ).set_env("LLAMA_ARG_CHECKPOINT_MIN_SPACING_NT").set_examples({LLAMA_EXAMPLE_SERVER})); add_opt(common_arg( diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 9dd7a34d11b6..abdd7c398b6a 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -1243,6 +1243,15 @@ struct server_context_impl { n_ctx_slot = n_ctx_train; } + // Continuum fork: `--checkpoint-min-step auto` spaces the checkpoints from the window this + // engine actually holds (after --fit), so they cover a whole slot; a caller never derives it + // from a window it guessed before launch. + if (params_base.checkpoint_min_step < 0) { + params_base.checkpoint_min_step = params_base.n_ctx_checkpoints > 0 + ? (n_ctx_slot + params_base.n_ctx_checkpoints - 1) / params_base.n_ctx_checkpoints + : 0; + } + slots.clear(); ctx_tgt_seq_rm_type = common_context_can_seq_rm(ctx_tgt); @@ -1802,7 +1811,12 @@ struct server_context_impl { } // the per-request limit takes priority over the global one - slot.n_predict_max = task.params.n_predict != -1 ? task.params.n_predict : params_base.n_predict; + // Continuum fork: an uncapped request decodes at most its slot's window, never on forever + // (continuum card 912e606d: with context shift, an uncapped request held a lane decoding 27k + // tokens). The launch no longer passes -n derived from a window it guessed. + slot.n_predict_max = task.params.n_predict != -1 ? task.params.n_predict + : params_base.n_predict != -1 ? params_base.n_predict + : slot.n_ctx; slot.task = std::make_unique(std::move(task)); From a41cfa53edfb5c10220450f2825d60d3fab471a8 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Tue, 3 Nov 2026 19:44:33 -0600 Subject: [PATCH 2/4] --fit-budget MiB0,...: the fit keeps at least (free - budget) of each device free, so the engine stays within the memory its node grants Continuum's memory authority grants serving a budget; the engine's own fit decides what fits inside it (continuum card 4a20255d, ONE-BUDGET 3b). Budgets are per device like --fit-target, a single value broadcast; empty means none (upstream behaviour). Threaded through common_fit_params (server init, llama-fit-params; llama-bench passes none). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- common/arg.cpp | 15 +++++++++++++++ common/common.cpp | 1 + common/common.h | 1 + common/fit.cpp | 18 ++++++++++++++++-- common/fit.h | 3 ++- tools/fit-params/fit-params.cpp | 3 ++- tools/llama-bench/llama-bench.cpp | 1 + 7 files changed, 38 insertions(+), 4 deletions(-) diff --git a/common/arg.cpp b/common/arg.cpp index 7148cf5af519..f485c9e6b0df 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -2903,6 +2903,21 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } } ).set_env("LLAMA_ARG_FIT_TARGET")); + add_opt(common_arg( + { "--fit-budget" }, "MiB0,MiB1,MiB2,...", + "the memory the engine may use per device for --fit (Continuum fork): the fit keeps at least (free - budget) of each device " + "free, so the engine stays within the budget its node's memory authority grants; a single value is broadcast across all devices", + [](common_params & params, const std::string & value) { + const std::vector split_arg = string_split(value, ','); + if (split_arg.size() > llama_max_devices()) { + throw std::invalid_argument(string_format("got %zu budgets but only %zu devices are supported", split_arg.size(), llama_max_devices())); + } + params.fit_params_budget.assign(llama_max_devices(), std::stoull(split_arg[0]) * 1024*1024); + for (size_t i = 1; i < split_arg.size(); i++) { + params.fit_params_budget[i] = std::stoull(split_arg[i]) * 1024*1024; + } + } + ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_FIT_PARAMS})); add_opt(common_arg( { "-fitc", "--fit-ctx" }, "N", string_format("minimum ctx size that can be set by --fit option, default: %" PRIu32, params.fit_params_min_ctx), diff --git a/common/common.cpp b/common/common.cpp index bda497246329..e95a1862b8d7 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1322,6 +1322,7 @@ common_init_result::common_init_result(common_params & params, bool model_only) params.tensor_split, params.tensor_buft_overrides.data(), params.fit_params_target.data(), + params.fit_params_budget.empty() ? nullptr : params.fit_params_budget.data(), params.fit_params_min_ctx, has_draft || spec_mtp ? &extra : nullptr, params.verbosity >= LOG_LEVEL_DEBUG ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_ERROR); diff --git a/common/common.h b/common/common.h index 42a259bf5502..1fb343b93eca 100644 --- a/common/common.h +++ b/common/common.h @@ -472,6 +472,7 @@ struct common_params { // margin per device in bytes for fitting parameters to free memory: std::vector fit_params_target = std::vector(llama_max_devices(), 1024 * 1024*1024); + std::vector fit_params_budget; // Continuum fork: bytes the engine may use per device (--fit-budget); empty = no budget enum llama_split_mode split_mode = LLAMA_SPLIT_MODE_LAYER; // how to split the model across GPUs enum llama_load_mode load_mode = LLAMA_LOAD_MODE_AUTO; // how to load the model diff --git a/common/fit.cpp b/common/fit.cpp index c601fe405ea5..27d4b9d1d1b4 100644 --- a/common/fit.cpp +++ b/common/fit.cpp @@ -178,7 +178,7 @@ common_device_memory_data_vec common_get_device_memory_data( static void common_params_fit_impl( const char * path_model, struct llama_model_params * mparams, struct llama_context_params * cparams, float * tensor_split, struct llama_model_tensor_buft_override * tensor_buft_overrides, - size_t * margins_s, uint32_t n_ctx_min, const common_fit_extra_model * extra, enum ggml_log_level log_level) { + size_t * margins_s, const size_t * budgets, uint32_t n_ctx_min, const common_fit_extra_model * extra, enum ggml_log_level log_level) { if (mparams->split_mode == LLAMA_SPLIT_MODE_TENSOR) { throw common_params_fit_exception("llama_params_fit is not implemented for SPLIT_MODE_TENSOR, abort"); } @@ -289,6 +289,19 @@ static void common_params_fit_impl( } } + // Continuum fork: a budget is the memory the engine may use on a device, so the fit leaves at + // least (free - budget) of it free; the node's memory authority decides the budget, the fit + // decides what fits inside it. + if (budgets != nullptr) { + if (nd == 0) { + margins[0] = std::max(margins[0], int64_t(dmds_full.back().total) - int64_t(budgets[0])); + } else { + for (size_t id = 0; id < nd; id++) { + margins[id] = std::max(margins[id], int64_t(dmds_full[id].free) - int64_t(budgets[id])); + } + } + } + std::vector dev_names; { dev_names.reserve(nd); @@ -882,13 +895,14 @@ enum common_params_fit_status common_fit_params( float * tensor_split, llama_model_tensor_buft_override * tensor_buft_overrides, size_t * margins, + const size_t * budgets, uint32_t n_ctx_min, const common_fit_extra_model * extra, ggml_log_level log_level) { const int64_t t0_us = llama_time_us(); common_params_fit_status status = COMMON_PARAMS_FIT_STATUS_SUCCESS; try { - common_params_fit_impl(path_model, mparams, cparams, tensor_split, tensor_buft_overrides, margins, n_ctx_min, extra, log_level); + common_params_fit_impl(path_model, mparams, cparams, tensor_split, tensor_buft_overrides, margins, budgets, n_ctx_min, extra, log_level); LOG_TRC("%s: successfully fit params to free device memory\n", __func__); } catch (const common_params_fit_exception & e) { LOG_WRN("%s: failed to fit params to free device memory: %s\n", __func__, e.what()); diff --git a/common/fit.h b/common/fit.h index 824d386b07a1..e0ef02051963 100644 --- a/common/fit.h +++ b/common/fit.h @@ -32,7 +32,8 @@ common_params_fit_status common_fit_params( llama_context_params * cparams, float * tensor_split, // writable buffer for tensor split, needs at least llama_max_devices elements llama_model_tensor_buft_override * tensor_buft_overrides, // writable buffer for overrides, needs at least llama_max_tensor_buft_overrides elements - size_t * margins, // margins of memory to leave per device in bytes + size_t * margins, + const size_t * budgets, // Continuum fork: bytes the engine may use per device, nullptr = no budget // margins of memory to leave per device in bytes uint32_t n_ctx_min, // minimum context size to set when trying to reduce memory use const common_fit_extra_model * extra, // model to fit alongside the main one, nullptr if there is none ggml_log_level log_level); // minimum log level to print during fitting, lower levels go to debug log diff --git a/tools/fit-params/fit-params.cpp b/tools/fit-params/fit-params.cpp index 20ca0be716a1..db8239616e90 100644 --- a/tools/fit-params/fit-params.cpp +++ b/tools/fit-params/fit-params.cpp @@ -34,7 +34,8 @@ int llama_fit_params(int argc, char ** argv) { if (!params.fit_params_print) { const common_params_fit_status status = common_fit_params(params.model.path.c_str(), &mparams, &cparams, - params.tensor_split, params.tensor_buft_overrides.data(), params.fit_params_target.data(), params.fit_params_min_ctx, + params.tensor_split, params.tensor_buft_overrides.data(), params.fit_params_target.data(), + params.fit_params_budget.empty() ? nullptr : params.fit_params_budget.data(), params.fit_params_min_ctx, nullptr, params.verbosity >= LOG_LEVEL_DEBUG ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_ERROR); if (params.fit_params_json) { diff --git a/tools/llama-bench/llama-bench.cpp b/tools/llama-bench/llama-bench.cpp index a2da93b9a282..8843d5ae1761 100644 --- a/tools/llama-bench/llama-bench.cpp +++ b/tools/llama-bench/llama-bench.cpp @@ -2293,6 +2293,7 @@ int llama_bench(int argc, char ** argv) { fit_tensor_split.data(), fit_overrides.data(), margins.data(), + nullptr, // no budget: the bench fits to free memory inst.fit_min_ctx, nullptr, params.verbose ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_ERROR); From 686dc270641a71e9e74e38cc03439998e408421e Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Tue, 3 Nov 2026 23:17:44 -0600 Subject: [PATCH 3/4] --checkpoint-min-step auto spaces each sequence's checkpoints from its own prompt, never a slot window; the uncapped-decode default is dropped (self-review on #94) Under --kv-unified a slot's window is the whole pool, so the first 'auto' (pool / --ctx-checkpoints) was 8,192 on a 262,144 pool: the spacing that measured cache_n 0 on every ~5k-token turn (continuum, 2026-09-13). Now step = ceil(n_prompt / n_ctx_checkpoints) per sequence (server_context::checkpoint_step), used at all three spacing sites. n_predict_max = slot.n_ctx is dropped: under one pool it let one uncapped request decode the whole pool. The bound on an uncapped decode is the cells a sequence was admitted to hold (continuum ONE-BUDGET 3b, card a156ee65), not a window. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- common/arg.cpp | 4 +-- tools/server/server-context.cpp | 47 +++++++++++++++++++-------------- 2 files changed, 29 insertions(+), 22 deletions(-) diff --git a/common/arg.cpp b/common/arg.cpp index f485c9e6b0df..9bf0a83c825b 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -1693,10 +1693,10 @@ common_params_context common_params_parser_init(common_params & params, llama_ex ).set_env("LLAMA_ARG_CTX_CHECKPOINTS").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); add_opt(common_arg( {"-cms", "--checkpoint-min-step"}, "N|auto", - string_format("minimum spacing between context checkpoints in tokens (default: %d, 0 = no minimum, auto = the slot's window over --ctx-checkpoints, so the checkpoints cover a whole slot)", params.checkpoint_min_step), + string_format("minimum spacing between context checkpoints in tokens (default: %d, 0 = no minimum, auto = each sequence's prompt over --ctx-checkpoints, so its checkpoints cover its whole prompt)", params.checkpoint_min_step), [](common_params & params, const std::string & value) { if (value == "auto") { - params.checkpoint_min_step = -1; // derived from the engine's own slot window once it is known + params.checkpoint_min_step = -1; // per sequence, from its own prompt (server_context::checkpoint_step) return; } const int n = std::stoi(value); diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index abdd7c398b6a..e80169b34519 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -1243,15 +1243,6 @@ struct server_context_impl { n_ctx_slot = n_ctx_train; } - // Continuum fork: `--checkpoint-min-step auto` spaces the checkpoints from the window this - // engine actually holds (after --fit), so they cover a whole slot; a caller never derives it - // from a window it guessed before launch. - if (params_base.checkpoint_min_step < 0) { - params_base.checkpoint_min_step = params_base.n_ctx_checkpoints > 0 - ? (n_ctx_slot + params_base.n_ctx_checkpoints - 1) / params_base.n_ctx_checkpoints - : 0; - } - slots.clear(); ctx_tgt_seq_rm_type = common_context_can_seq_rm(ctx_tgt); @@ -1372,8 +1363,13 @@ struct server_context_impl { SRV_TRC("%s", "for more info see https://github.com/ggml-org/llama.cpp/pull/16391\n"); if (params_base.n_ctx_checkpoints > 0) { - SRV_TRC("context checkpoints enabled, max = %d, min spacing = %d\n", - params_base.n_ctx_checkpoints, params_base.checkpoint_min_step); + if (params_base.checkpoint_min_step < 0) { + SRV_TRC("context checkpoints enabled, max = %d, min spacing = each prompt over %d\n", + params_base.n_ctx_checkpoints, params_base.n_ctx_checkpoints); + } else { + SRV_TRC("context checkpoints enabled, max = %d, min spacing = %d\n", + params_base.n_ctx_checkpoints, params_base.checkpoint_min_step); + } } else { SRV_TRC("%s", "context checkpoints disabled\n"); } @@ -1811,12 +1807,7 @@ struct server_context_impl { } // the per-request limit takes priority over the global one - // Continuum fork: an uncapped request decodes at most its slot's window, never on forever - // (continuum card 912e606d: with context shift, an uncapped request held a lane decoding 27k - // tokens). The launch no longer passes -n derived from a window it guessed. - slot.n_predict_max = task.params.n_predict != -1 ? task.params.n_predict - : params_base.n_predict != -1 ? params_base.n_predict - : slot.n_ctx; + slot.n_predict_max = task.params.n_predict != -1 ? task.params.n_predict : params_base.n_predict; slot.task = std::make_unique(std::move(task)); @@ -2476,15 +2467,31 @@ struct server_context_impl { return true; } + // Continuum fork: the minimum spacing between this sequence's checkpoints. A fixed + // `--checkpoint-min-step` is used as given; `auto` spaces them so `--ctx-checkpoints` of them + // cover this sequence's own prompt, never a slot window: under `--kv-unified` a slot's window + // is the whole pool, and pool / 32 (8,192 on a 262,144 pool) skipped every divergence point + // of a ~5k-token turn (continuum, measured cache_n 0 on every turn, 2026-09-13). + int64_t checkpoint_step(const server_slot & slot) const { + if (params_base.checkpoint_min_step >= 0) { + return params_base.checkpoint_min_step; + } + if (params_base.n_ctx_checkpoints <= 0) { + return 0; + } + return (slot.task->n_tokens() + params_base.n_ctx_checkpoints - 1) / params_base.n_ctx_checkpoints; + } + // n_tokens_cur: the number of tokens added to the batch for the current slot void create_checkpoint(server_slot & slot, const int64_t n_tokens_cur, llama_pos pos_min, llama_pos pos_max) { const int id_task = slot.task->id; // evict checkpoints within min-step of a previous checkpoint, unless they were // created by the current task + const int64_t step = checkpoint_step(slot); int64_t last = -1; for (auto it = slot.prompt.checkpoints.begin(); it != slot.prompt.checkpoints.end(); ) { - if (it->id_task != id_task && last >= 0 && it->n_tokens <= last + params_base.checkpoint_min_step) { + if (it->id_task != id_task && last >= 0 && it->n_tokens <= last + step) { SLT_TRC(slot, "erasing context checkpoint too close to an earlier one (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", size = %.3f MiB)\n", it->pos_min, it->pos_max, it->n_tokens, (float) it->size() / 1024 / 1024); @@ -4199,7 +4206,7 @@ struct server_context_impl { const auto pos = slot.prompt.n_tokens(); const auto & checkpoints = slot.prompt.checkpoints; - if (pos == last_user_pos || checkpoints.empty() || pos > checkpoints.back().n_tokens + params_base.checkpoint_min_step) { + if (pos == last_user_pos || checkpoints.empty() || pos > checkpoints.back().n_tokens + checkpoint_step(slot)) { break; } } @@ -4283,7 +4290,7 @@ struct server_context_impl { do_checkpoint = do_checkpoint && ( slot.prompt.checkpoints.empty() || is_last_user_message || near_prompt_end || is_view_end || - n_tokens_start > slot.prompt.checkpoints.back().n_tokens + params_base.checkpoint_min_step); + n_tokens_start > slot.prompt.checkpoints.back().n_tokens + checkpoint_step(slot)); SLT_DBG(slot, "main/do_checkpoint = %s, pos_min = %d, pos_max = %d\n", do_checkpoint ? "yes" : "no", pos_min, pos_max); // note: we create the checkpoint before calling llama_decode(), so the current batch is not From cde5c2f160bd3b722803b4b12def2386e7c38e00 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Tue, 3 Nov 2026 23:22:26 -0600 Subject: [PATCH 4/4] --fit-budget refuses a negative value instead of wrapping it into no budget; fit.h's margins comment back on its line (Cormac on #94) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- common/arg.cpp | 11 +++++++++-- common/fit.h | 4 ++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/common/arg.cpp b/common/arg.cpp index 9bf0a83c825b..4353464d2847 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -2912,9 +2912,16 @@ common_params_context common_params_parser_init(common_params & params, llama_ex if (split_arg.size() > llama_max_devices()) { throw std::invalid_argument(string_format("got %zu budgets but only %zu devices are supported", split_arg.size(), llama_max_devices())); } - params.fit_params_budget.assign(llama_max_devices(), std::stoull(split_arg[0]) * 1024*1024); + // stoull wraps a leading '-' into a huge value, which would silently mean no budget + const auto mib = [](const std::string & v) -> size_t { + if (v.empty() || v[0] == '-') { + throw std::invalid_argument("--fit-budget takes non-negative MiB values, got '" + v + "'"); + } + return std::stoull(v) * 1024*1024; + }; + params.fit_params_budget.assign(llama_max_devices(), mib(split_arg[0])); for (size_t i = 1; i < split_arg.size(); i++) { - params.fit_params_budget[i] = std::stoull(split_arg[i]) * 1024*1024; + params.fit_params_budget[i] = mib(split_arg[i]); } } ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_FIT_PARAMS})); diff --git a/common/fit.h b/common/fit.h index e0ef02051963..f912ae57e720 100644 --- a/common/fit.h +++ b/common/fit.h @@ -32,8 +32,8 @@ common_params_fit_status common_fit_params( llama_context_params * cparams, float * tensor_split, // writable buffer for tensor split, needs at least llama_max_devices elements llama_model_tensor_buft_override * tensor_buft_overrides, // writable buffer for overrides, needs at least llama_max_tensor_buft_overrides elements - size_t * margins, - const size_t * budgets, // Continuum fork: bytes the engine may use per device, nullptr = no budget // margins of memory to leave per device in bytes + size_t * margins, // margins of memory to leave per device in bytes + const size_t * budgets, // Continuum fork: bytes the engine may use per device, nullptr = no budget uint32_t n_ctx_min, // minimum context size to set when trying to reduce memory use const common_fit_extra_model * extra, // model to fit alongside the main one, nullptr if there is none ggml_log_level log_level); // minimum log level to print during fitting, lower levels go to debug log