From 78f5717dc4c5f411b44844ebd8305a2f5d561ad7 Mon Sep 17 00:00:00 2001 From: Ric Lewis Date: Mon, 19 Oct 2026 22:49:39 -0700 Subject: [PATCH] [Engine] allow CPU vision to load on demand without elastic VRAM --- docs/DETAILS.md | 4 +++ docs/VRAM_ELASTIC.md | 6 ++++ serve/server.py | 33 +++++++++++++------- serve/test_vision_elastic.py | 60 ++++++++++++++++++++++++++++++++++-- 4 files changed, 90 insertions(+), 13 deletions(-) diff --git a/docs/DETAILS.md b/docs/DETAILS.md index 0fddc11e2e0..b7e6bc9c5bc 100644 --- a/docs/DETAILS.md +++ b/docs/DETAILS.md @@ -965,6 +965,10 @@ every turn, is encoded once. On an NVIDIA card under 24 GB the CPU encoder is the default, because the GPU encoder takes ~850 expert slots of a 16 GB card's ~4,500 (about 5% of decode). `--vision gpu` is kept with a note. +For a CPU encoder (`"gpu": false`), `"elastic": true` uses the same on-demand start and idle stop without +`"vram_elastic"` or any VRAM reserve; text requests leave it unloaded. See the configuration in +[VRAM_ELASTIC.md](VRAM_ELASTIC.md#the-image-encoder-on-demand). + ### Sending a picture Terminal chat: type `/image `, press Enter, then type your question. OpenAI API: an `image_url` part (a `data:` URL, an diff --git a/docs/VRAM_ELASTIC.md b/docs/VRAM_ELASTIC.md index 94945094d34..dabf61fc476 100644 --- a/docs/VRAM_ELASTIC.md +++ b/docs/VRAM_ELASTIC.md @@ -107,6 +107,12 @@ expert outside the cache), but I have not run a fault-injection test of these tw ## The image encoder on demand +For a CPU encoder, set `"gpu": false` and `"elastic": true` in the existing `"vision"` section. It starts with the +first picture and stops after `"elastic_idle_s"` (default 120) without one, between requests. Text requests do not +start it. CPU on-demand loading needs no `"vram_elastic"`, reserves no VRAM and leaves the expert cache alone. +The next picture pays the encoder's load time again; unloading ends its process, but the OS may keep reclaimable +file-cache pages. Without `"elastic": true`, startup is unchanged. + With `"elastic": true` in the config's `"vision"` section (and `"vram_elastic": true`; without it the server says so and starts the encoder with the model, as before) the GPU encoder is not started with the server or with a load: diff --git a/serve/server.py b/serve/server.py index 5614dbf1eaf..a7dfe08789a 100644 --- a/serve/server.py +++ b/serve/server.py @@ -2663,6 +2663,11 @@ class Vision: lines and writes each image's embeddings; results are cached by the image's hash, so a conversation that sends the same picture again (every turn, with most clients) encodes it once.""" + @staticmethod + def elastic_enabled(cfg: dict, vram_elastic: bool = False) -> bool: + """CPU on-demand loading needs no elastic VRAM cache; GPU loading still does.""" + return cfg.get("elastic") is True and (not cfg.get("gpu") or vram_elastic is True) + @staticmethod def work_dir() -> Path: """#480: the directory for the encoder's files. The engine reads the embeddings file named in a `GENI` line @@ -2695,9 +2700,11 @@ def __init__(self, cfg: dict, log=None, env: dict | None = None, lazy: bool = Fa self.stopped = True self.lock = threading.Lock() self.cache: dict[str, tuple[Path, int]] = {} - # opt-in (the vision section's "elastic": true, with the config's "vram_elastic"): the encoder starts with the - # first picture and stops after elastic_idle_s without one; while it runs, the engine's expert cache keeps - # elastic_mib of VRAM free for it (Service._vision_up / _vision_idle_down) + self.gpu = bool(cfg.get("gpu")) + # opt-in (the vision section's "elastic": true): the encoder starts with the first picture and stops after + # elastic_idle_s without one. A GPU encoder also needs the config's "vram_elastic": while it runs, the engine's + # expert cache keeps elastic_mib of VRAM free for it (Service._vision_up / Service._vision_idle_down). A CPU + # encoder holds no VRAM, so it starts on demand without it. self.max_tokens = self.token_limit(cfg) self.elastic = cfg.get("elastic") is True self.elastic_mib = int(cfg.get("elastic_mib") or 2048) @@ -4340,14 +4347,15 @@ def _engine_reserve(self) -> int | None: return self.vram_reserve if self.vram_reserve_asked else 0 def _vision_up(self): - """The image turn's start for an elastic encoder that is not running (the caller holds self.fifo): the expert - cache first keeps the encoder's elastic_mib of VRAM free on top of the reserve, then the encoder starts (load and + """Start an on-demand encoder for an image (the caller holds self.fifo). CPU starts without touching VRAM. + For GPU, the expert cache first keeps elastic_mib of VRAM free on top of the reserve, then the encoder starts (load and warm-up, a few seconds). If the cache cannot shrink (an engine without --vram-elastic, the resident low-RAM mode) the encoder starts anyway, as a non-elastic one would.""" v = self.vision if not getattr(v, "elastic", False) or v.alive(): return - if hasattr(self.engine, "vram") and self.loaded(): + gpu = getattr(v, "gpu", True) + if gpu and hasattr(self.engine, "vram") and self.loaded(): try: out = self.engine.vram((self._engine_reserve() or 0) + v.elastic_mib) self.vision_held = True @@ -4358,12 +4366,12 @@ def _vision_up(self): "it anyway", flush=True) if self.vision_held: time.sleep(0.2) # the driver's free-VRAM counter shows the cache's release a moment later - free0, t0 = self.free_vram_mib(), time.time() + free0, t0 = (self.free_vram_mib() if gpu else None), time.time() try: v.restart() except Exception as e: # a 503 as for a non-elastic encoder at a load, not a dropped connection raise EngineDied(f"the image encoder failed to start: {e}") from e - free1 = self.free_vram_mib() + free1 = self.free_vram_mib() if gpu else None took = f", {free0 - free1} MiB of VRAM" if free0 is not None and free1 is not None else "" print(f"[strata] images: the image encoder started in {time.time() - t0:.1f} s{took}", flush=True) @@ -4385,6 +4393,8 @@ def _vision_idle_down(self, force: bool = False) -> str: out = self.engine.vram(self._engine_reserve()) back = (f"; the expert cache took its VRAM back ({out.get('expert_cache_mib')} of " f"{out.get('expert_cache_full_mib')} MiB)") + elif not getattr(v, "gpu", True): + back = "; the CPU encoder holds no VRAM" # also when the cache never made room (a layer split, an engine without --vram-elastic): the stop happened print(f"[strata] images: the image encoder stopped after {v.elastic_idle_s:g} s without a picture{back}", flush=True) @@ -4398,7 +4408,8 @@ def start_vision_idle(self): if not getattr(v, "elastic", False): return print(f"[strata] images: the image encoder starts with the first picture and stops after {v.elastic_idle_s:g} s " - f"without one; while it runs the expert cache keeps {v.elastic_mib} MiB free for it", flush=True) + + (f"without one; while it runs the expert cache keeps {v.elastic_mib} MiB free for it" + if getattr(v, "gpu", True) else "without one (CPU; no VRAM reserve)"), flush=True) def loop(): while True: @@ -8949,7 +8960,7 @@ def request_stop(reason: str): for line in card_roles_lines(cfg): # the card roles: said first; every card Off stops here, before anything touches a card print(line, flush=True) if cfg.get("vision"): - elastic = cfg["vision"].get("elastic") is True and cfg.get("vram_elastic") is True + elastic = Vision.elastic_enabled(cfg["vision"], cfg.get("vram_elastic")) print("the vision encoder starts with the first picture (elastic) ..." if elastic else "loading the vision encoder ..." if not lazy else "vision encoder unloaded; it starts with the model ...", flush=True) @@ -8959,7 +8970,7 @@ def request_stop(reason: str): for k, v in cfg["vision"].items() if not (k == "exe" and engine_component.is_reference(v))} if engine_component.is_reference(cfg["vision"].get("exe")): # the engine from the component folder vcfg["exe"] = _component_exe(cfg["vision"]["exe"], cfg) - if vcfg.get("elastic") is True and cfg.get("vram_elastic") is not True: + if vcfg.get("elastic") is True and not elastic: print('[strata] the vision section\'s "elastic" needs "vram_elastic": true (the expert cache gives the ' "encoder its VRAM): the encoder starts with the model instead", flush=True) vcfg["elastic"] = False diff --git a/serve/test_vision_elastic.py b/serve/test_vision_elastic.py index fa16426cda0..1f2ba1a959f 100644 --- a/serve/test_vision_elastic.py +++ b/serve/test_vision_elastic.py @@ -1,4 +1,4 @@ -"""The elastic image encoder (the vision section's "elastic": true with "vram_elastic"): the first picture makes the +"""The on-demand CPU encoder and elastic GPU encoder ("elastic": true, plus "vram_elastic" for GPU): the first picture makes the expert cache keep the encoder's VRAM free and then starts it, later pictures use it as it is, an idle encoder stops and the cache takes its VRAM back, a POST /v1/vram reserve stacks on the encoder's share, and the engine's load never starts it on its own.""" @@ -8,6 +8,7 @@ import time import unittest from pathlib import Path +from unittest import mock from serve.frontend import ChatTemplate from serve.server import ByteTokenizer, EngineDied, Service, Vision @@ -19,7 +20,8 @@ class ElasticVision: """An encoder process stand-in: starts, stops, and turns every picture into three image tokens.""" - def __init__(self, d, elastic=True): + def __init__(self, d, elastic=True, gpu=True): + self.gpu = gpu self.elastic, self.elastic_mib, self.elastic_idle_s = elastic, 2048, 60.0 self.up, self.starts, self.unloads = False, 0, 0 self.dir = Path(d) @@ -141,6 +143,60 @@ def test_the_config_keys(self): self.assertFalse(v.elastic) self.assertEqual((v.elastic_mib, v.elastic_idle_s), (2048, 120.0)) + def test_cpu_text_then_images_idle_and_reload_never_touch_vram(self): + self.vision.gpu = False + self.engine.loaded = False + with mock.patch.object(self.engine, "vram", side_effect=AssertionError("CPU vision resized VRAM")), \ + mock.patch.object(self.svc, "free_vram_mib", side_effect=AssertionError("CPU vision queried VRAM")): + self.svc.ensure_loaded() # first text request loads only the text engine + self.svc.prepare([{"role": "user", "content": "hello"}], None, {}) + self.assertEqual(self.engine.starts, 1) + self.assertEqual(self.vision.starts, 0) + self.assertFalse(self.vision.alive()) + self.picture() + self.assertEqual(self.vision.starts, 1) + self.assertFalse(self.svc.vision_held) + self.picture() # reuse while warm + self.assertEqual(self.vision.starts, 1) + self.assertEqual(self.svc._vision_idle_down(), "busy") + self.svc.last_image_at = time.time() - 61 + with self.svc.fifo: + self.assertEqual(self.svc._vision_idle_down(), "busy") + self.assertEqual(self.svc._vision_idle_down(), "down") + self.assertFalse(self.vision.alive()) + self.assertEqual(self.vision.unloads, 1) + self.assertEqual(self.svc._vision_idle_down(), "nothing") + self.svc.ensure_loaded() # text after idle still leaves vision down + self.assertEqual(self.vision.starts, 1) + self.picture() + self.assertEqual(self.vision.starts, 2) + self.svc.unload() + self.assertFalse(self.vision.alive()) + + def test_cpu_opt_in_and_gpu_gate(self): + for gpu in (False, True): + for elastic in (False, True): + for vram in (False, True): + with self.subTest(gpu=gpu, elastic=elastic, vram=vram): + self.assertEqual(Vision.elastic_enabled({"gpu": gpu, "elastic": elastic}, vram), + elastic and (not gpu or vram)) + self.assertFalse(Vision.elastic_enabled({"gpu": False, "elastic": "yes"})) + self.assertFalse(Vision.elastic_enabled({"gpu": True, "elastic": True}, "yes")) + + def test_cpu_startup_is_eager_unless_elastic_or_lazy(self): + cfg = {"exe": "e", "mmproj": "m", "model": "t", "gpu": False} + with mock.patch.object(Vision, "_start") as start: + v = Vision(cfg) + self.assertFalse(v.gpu or v.elastic) + start.assert_called_once_with() + start.reset_mock() + v = Vision({**cfg, "elastic": True}) + self.assertTrue(v.elastic) + self.assertFalse(v.gpu) + start.assert_not_called() + Vision(cfg, lazy=True) + start.assert_not_called() + if __name__ == "__main__": unittest.main()