[Engine] allow CPU vision to load on demand without elastic VRAM

docs/DETAILS.md

@@ -1725,6 +1725,10 @@ took 1,339 MiB and started in 4.7 s (2.7 s the next time); a request with one pi
 5.0 s on a later cold start, and 2 / 5 / 10 pictures with the encoder running took 3.8 / 5.6 / 9.0 s (~0.45 s to encode
 each, the same as on a second card). Without the key nothing changes.
 
+For a CPU encoder (`"gpu": false`), `"elastic": true` uses the same on-demand start and idle stop without
+`"vram_elastic"` or any VRAM reserve; text requests leave it unloaded. See the configuration in
+[VRAM_ELASTIC.md](VRAM_ELASTIC.md#the-image-encoder-on-demand).
+
 ### Sending a picture
 
 **Terminal chat:** type `/image <path to a picture>`, press Enter, then type your question.

docs/VRAM_ELASTIC.md

@@ -107,6 +107,12 @@ expert outside the cache), but I have not run a fault-injection test of these tw
 
 ## The image encoder on demand
 
+For a CPU encoder, set `"gpu": false` and `"elastic": true` in the existing `"vision"` section. It starts with the
+first picture and stops after `"elastic_idle_s"` (default 120) without one, between requests. Text requests do not
+start it. CPU on-demand loading needs no `"vram_elastic"`, reserves no VRAM and leaves the expert cache alone.
+The next picture pays the encoder's load time again; unloading ends its process, but the OS may keep reclaimable
+file-cache pages. Without `"elastic": true`, startup is unchanged.
+
 With `"elastic": true` in the config's `"vision"` section (and `"vram_elastic": true`; without it the server says so
 and starts the encoder with the model, as before) the GPU encoder is not started with the server or with a load:
 

serve/server.py

@@ -2512,6 +2512,11 @@ class Vision:
     lines and writes each image's embeddings; results are cached by the image's hash, so a conversation that
     sends the same picture again (every turn, with most clients) encodes it once."""
 
+    @staticmethod
+    def elastic_enabled(cfg: dict, vram_elastic: bool = False) -> bool:
+        """CPU on-demand loading needs no elastic VRAM cache; GPU loading still does."""
+        return cfg.get("elastic") is True and (not cfg.get("gpu") or vram_elastic is True)
+
     @staticmethod
     def work_dir() -> Path:
         """#480: the directory for the encoder's files.  The engine reads the embeddings file named in a `GENI` line
@@ -2544,9 +2549,9 @@ def __init__(self, cfg: dict, log=None, env: dict | None = None, lazy: bool = Fa
         self.stopped = True
         self.lock = threading.Lock()
         self.cache: dict[str, tuple[Path, int]] = {}
-        # opt-in (the vision section's "elastic": true, with the config's "vram_elastic"): the encoder starts with the
-        # first picture and stops after elastic_idle_s without one; while it runs, the engine's expert cache keeps
-        # elastic_mib of VRAM free for it (Service._vision_up / _vision_idle_down)
+        self.gpu = bool(cfg.get("gpu"))
+        # opt-in (the vision section's "elastic": true): start with the first picture and stop after elastic_idle_s.
+        # GPU encoders also need "vram_elastic"; the cache keeps elastic_mib free while they run. CPU needs no VRAM.
         self.elastic = cfg.get("elastic") is True
         self.elastic_mib = int(cfg.get("elastic_mib") or 2048)
         self.elastic_idle_s = float(cfg.get("elastic_idle_s") or 120)
@@ -3820,14 +3825,15 @@ def _engine_reserve(self) -> int | None:
         return self.vram_reserve if self.vram_reserve_asked else 0
 
     def _vision_up(self):
-        """The image turn's start for an elastic encoder that is not running (the caller holds self.fifo): the expert
-        cache first keeps the encoder's elastic_mib of VRAM free on top of the reserve, then the encoder starts (load and
+        """Start an on-demand encoder for an image (the caller holds self.fifo). CPU starts without touching VRAM.
+        For GPU, the expert cache first keeps elastic_mib of VRAM free on top of the reserve, then the encoder starts (load and
         warm-up, a few seconds).  If the cache cannot shrink (an engine without --vram-elastic, the resident low-RAM mode)
         the encoder starts anyway, as a non-elastic one would."""
         v = self.vision
         if not getattr(v, "elastic", False) or v.alive():
             return
-        if hasattr(self.engine, "vram") and self.loaded():
+        gpu = getattr(v, "gpu", True)
+        if gpu and hasattr(self.engine, "vram") and self.loaded():
             try:
                 out = self.engine.vram((self._engine_reserve() or 0) + v.elastic_mib)
                 self.vision_held = True
@@ -3838,9 +3844,9 @@ def _vision_up(self):
                       "it anyway", flush=True)
         if self.vision_held:
             time.sleep(0.2)          # the driver's free-VRAM counter shows the cache's release a moment later
-        free0, t0 = self.free_vram_mib(), time.time()
+        free0, t0 = self.free_vram_mib() if gpu else None, time.time()
         v.restart()
-        free1 = self.free_vram_mib()
+        free1 = self.free_vram_mib() if gpu else None
         took = f", {free0 - free1} MiB of VRAM" if free0 is not None and free1 is not None else ""
         print(f"[strata] images: the image encoder started in {time.time() - t0:.1f} s{took}", flush=True)
 
@@ -3862,6 +3868,9 @@ def _vision_idle_down(self, force: bool = False) -> str:
                 print(f"[strata] images: the image encoder stopped after {v.elastic_idle_s:g} s without a picture; "
                       f"the expert cache took its VRAM back ({out.get('expert_cache_mib')} of "
                       f"{out.get('expert_cache_full_mib')} MiB)", flush=True)
+            elif not getattr(v, "gpu", True):
+                print(f"[strata] images: the CPU image encoder stopped after {v.elastic_idle_s:g} s without a picture",
+                      flush=True)
             self.vision_held = False
             return "down"
         finally:
@@ -3872,7 +3881,8 @@ def start_vision_idle(self):
         if not getattr(v, "elastic", False):
             return
         print(f"[strata] images: the image encoder starts with the first picture and stops after {v.elastic_idle_s:g} s "
-              f"without one; while it runs the expert cache keeps {v.elastic_mib} MiB free for it", flush=True)
+              + (f"without one; while it runs the expert cache keeps {v.elastic_mib} MiB free for it"
+                 if getattr(v, "gpu", True) else "without one (CPU; no VRAM reserve)"), flush=True)
 
         def loop():
             while True:
@@ -7924,7 +7934,7 @@ def request_stop(reason: str):
             print(f"[strata] sampling defaults from the config: {pretty}", flush=True)
         lazy = a.lazy or cfg.get("lazy_load") is True
         if cfg.get("vision"):
-            elastic = cfg["vision"].get("elastic") is True and cfg.get("vram_elastic") is True
+            elastic = Vision.elastic_enabled(cfg["vision"], cfg.get("vram_elastic"))
             print("the vision encoder starts with the first picture (elastic) ..." if elastic else
                   "loading the vision encoder ..." if not lazy else
                   "vision encoder unloaded; it starts with the model ...", flush=True)
@@ -7934,7 +7944,7 @@ def request_stop(reason: str):
                     for k, v in cfg["vision"].items() if not (k == "exe" and engine_component.is_reference(v))}
             if engine_component.is_reference(cfg["vision"].get("exe")):     # the engine from the component folder
                 vcfg["exe"] = _component_exe(cfg["vision"]["exe"], cfg)
-            if vcfg.get("elastic") is True and cfg.get("vram_elastic") is not True:
+            if vcfg.get("elastic") is True and not elastic:
                 print('[strata] the vision section\'s "elastic" needs "vram_elastic": true (the expert cache gives the '
                       "encoder its VRAM): the encoder starts with the model instead", flush=True)
                 vcfg["elastic"] = False

serve/test_vision_elastic.py

@@ -1,11 +1,12 @@
-"""The elastic image encoder (the vision section's "elastic": true with "vram_elastic"): the first picture makes the
+"""The on-demand CPU encoder and elastic GPU encoder ("elastic": true, plus "vram_elastic" for GPU): the first picture makes the
 expert cache keep the encoder's VRAM free and then starts it, later pictures use it as it is, an idle encoder stops and
 the cache takes its VRAM back, a POST /v1/vram reserve stacks on the encoder's share, and the engine's load never starts
 it on its own."""
 import tempfile
 import time
 import unittest
 from pathlib import Path
+from unittest import mock
 
 from serve.frontend import ChatTemplate
 from serve.server import ByteTokenizer, Service, Vision
@@ -17,7 +18,8 @@
 
 class ElasticVision:
     """An encoder process stand-in: starts, stops, and turns every picture into three image tokens."""
-    def __init__(self, d, elastic=True):
+    def __init__(self, d, elastic=True, gpu=True):
+        self.gpu = gpu
         self.elastic, self.elastic_mib, self.elastic_idle_s = elastic, 2048, 60.0
         self.up, self.starts, self.unloads = False, 0, 0
         self.dir = Path(d)
@@ -118,6 +120,60 @@ def test_the_config_keys(self):
         self.assertFalse(v.elastic)
         self.assertEqual((v.elastic_mib, v.elastic_idle_s), (2048, 120.0))
 
+    def test_cpu_text_then_images_idle_and_reload_never_touch_vram(self):
+        self.vision.gpu = False
+        self.engine.loaded = False
+        with mock.patch.object(self.engine, "vram", side_effect=AssertionError("CPU vision resized VRAM")), \
+                mock.patch.object(self.svc, "free_vram_mib", side_effect=AssertionError("CPU vision queried VRAM")):
+            self.svc.ensure_loaded()                       # first text request loads only the text engine
+            self.svc.prepare([{"role": "user", "content": "hello"}], None, {})
+            self.assertEqual(self.engine.starts, 1)
+            self.assertEqual(self.vision.starts, 0)
+            self.assertFalse(self.vision.alive())
+            self.picture()
+            self.assertEqual(self.vision.starts, 1)
+            self.assertFalse(self.svc.vision_held)
+            self.picture()                                 # reuse while warm
+            self.assertEqual(self.vision.starts, 1)
+            self.assertEqual(self.svc._vision_idle_down(), "busy")
+            self.svc.last_image_at = time.time() - 61
+            with self.svc.fifo:
+                self.assertEqual(self.svc._vision_idle_down(), "busy")
+            self.assertEqual(self.svc._vision_idle_down(), "down")
+            self.assertFalse(self.vision.alive())
+            self.assertEqual(self.vision.unloads, 1)
+            self.assertEqual(self.svc._vision_idle_down(), "nothing")
+            self.svc.ensure_loaded()                        # text after idle still leaves vision down
+            self.assertEqual(self.vision.starts, 1)
+            self.picture()
+            self.assertEqual(self.vision.starts, 2)
+            self.svc.unload()
+            self.assertFalse(self.vision.alive())
+
+    def test_cpu_opt_in_and_gpu_gate(self):
+        for gpu in (False, True):
+            for elastic in (False, True):
+                for vram in (False, True):
+                    with self.subTest(gpu=gpu, elastic=elastic, vram=vram):
+                        self.assertEqual(Vision.elastic_enabled({"gpu": gpu, "elastic": elastic}, vram),
+                                         elastic and (not gpu or vram))
+        self.assertFalse(Vision.elastic_enabled({"gpu": False, "elastic": "yes"}))
+        self.assertFalse(Vision.elastic_enabled({"gpu": True, "elastic": True}, "yes"))
+
+    def test_cpu_startup_is_eager_unless_elastic_or_lazy(self):
+        cfg = {"exe": "e", "mmproj": "m", "model": "t", "gpu": False}
+        with mock.patch.object(Vision, "_start") as start:
+            v = Vision(cfg)
+            self.assertFalse(v.gpu or v.elastic)
+            start.assert_called_once_with()
+            start.reset_mock()
+            v = Vision({**cfg, "elastic": True})
+            self.assertTrue(v.elastic)
+            self.assertFalse(v.gpu)
+            start.assert_not_called()
+            Vision(cfg, lazy=True)
+            start.assert_not_called()
+
 
 if __name__ == "__main__":
     unittest.main()