[WIP] Preset patching, per-trial records, and --previous

mkdocs/docs/concepts/presets.md

@@ -9,6 +9,8 @@ A preset configuration lets you use an agent to create a preset: a verified and
 
 The value of presets comes from combining two fundamental features: agent-driven model inference optimization and the `dstack` [service](services.md) primitive, which can deploy model inference to any cloud, Kubernetes, or on-prem cluster.
 
+To get the best performance for the given model, hardware, and other constraints, the agent selects the serving framework, quantization, and serving parameters, and can patch the framework's source code, generate custom kernels, and patch drivers.
+
 > The presets feature is experimental and may change.
 
 ??? info "Prerequisites"
@@ -169,6 +171,23 @@ prompt: |
 
 Set `baseline: true` to make the first trial a baseline: the agent serves the model the way the chosen serving framework recommends, without tuning it for performance. Later trials are optimization attempts.
 
+### Previous sessions
+
+Set `previous` to a list of preset IDs to give the agent the results of earlier creation sessions. It analyzes what they tried and how it worked, and aims to improve on them instead of rediscovering it.
+
+<div editor-title="preset.dstack.yml">
+
+```yaml
+previous:
+  - c83375b4
+```
+
+</div>
+
+Alternatively, pass `--previous` (repeatable) to `dstack preset create`.
+
+With `baseline: true`, the first trial reproduces the best comparable previous result to confirm it still holds before optimizing further.
+
 !!! info "Reference"
     The `preset` configuration supports many more options. See the [`.dstack.yml` reference](../reference/dstack.yml/preset.md).
 
@@ -251,13 +270,11 @@ $ dstack preset delete c83375b4
 
 For command options and agent settings, see the [`dstack preset` CLI reference](../reference/cli/dstack/preset.md).
 
-!!! info "Roadmap and feedback"
-    Here's what is coming soon:
-
-    * Allow the agent to change the source code, compile binaries, etc.
-    * Support for PD disaggregation
-    * Allow passing multiple `--previous <preset ID>` to `dstack preset create` to reuse the insights from previous sessions
-    * Allow passing ranges to `concurrency`
+!!! info "Limitations"
+    * Currently, the agent doesn't upload compiled binaries anywhere; patches compile at runtime
+    * Doesn't support PD disaggregation (coming soon)
+    * Doesn't allow a custom dataset; always uses `random`
+    * Doesn't support ranges for `concurrency`
 
     Report bugs and request features on [GitHub](https://github.com/dstackai/dstack/issues), and ask questions on [Discord](https://discord.gg/u8SmfwPpMd).
 

src/dstack/_internal/cli/commands/preset.py

@@ -22,6 +22,7 @@
     plan_preset,
     reassign_preset_name,
     reconcile_detached_sessions,
+    resolve_previous_sessions,
     show_preset_session_logs,
     stop_preset_session,
 )
@@ -114,6 +115,13 @@ def _register(self) -> None:
             action="store_true",
             help="Save the agent prompt and raw trace",
         )
+        create_parser.add_argument(
+            "--previous",
+            action="append",
+            metavar="ID",
+            help="Give the agent a previous session's results to analyze and improve on."
+            " Repeat for several",
+        )
         create_parser.add_argument(
             "--resume",
             metavar="ID",
@@ -286,6 +294,14 @@ def _create(self, args: argparse.Namespace) -> None:
                     "[warning]--trials is ignored when resuming: "
                     "the constraints are fixed at creation[/]"
                 )
+            if configuration.previous:
+                console.print(
+                    "[warning]previous is ignored when resuming: "
+                    "the previous sessions are fixed at creation[/]"
+                )
+        previous = ()
+        if resume_session is None and configuration.previous:
+            previous = resolve_previous_sessions(configuration.previous)
         api = Client.from_config(project_name=args.project)
         allowed_fleets = None
         if resume_session is None:
@@ -310,6 +326,7 @@ def _create(self, args: argparse.Namespace) -> None:
                 resume_session=resume_session,
                 user_prompt=user_prompt,
                 allowed_fleets=allowed_fleets,
+                previous=previous,
             )
         except KeyboardInterrupt:
             return  # the interrupt handler already reported detach / stop
@@ -524,6 +541,8 @@ def _get_effective_configuration(
     _apply_name(configuration, args.name, required=require_name)
     if getattr(args, "trials", None) is not None:
         configuration.trials = args.trials
+    if getattr(args, "previous", None):
+        configuration.previous = list(args.previous)
     profile = load_profile_from_args(args=args, repo_dir=Path.cwd())
     for field in ProfileParams.model_fields:
         if getattr(configuration, field) is None:

src/dstack/_internal/cli/models/configurations.py

@@ -154,6 +154,15 @@ class PresetConfiguration(
             )
         ),
     ] = None
+    previous: Annotated[
+        Optional[list[str]],
+        Field(
+            description=(
+                "The IDs of previous presets whose creation results the agent"
+                " analyzes and improves on"
+            )
+        ),
+    ] = None
     concurrency: Annotated[
         Optional[PositiveInt],
         Field(

src/dstack/_internal/cli/models/preset_agent.py

@@ -85,6 +85,7 @@
         "run_id": {"type": "string"},
         "run_name": {"type": "string"},
         "service_yaml": {"type": "string"},
+        "trial": {"type": "integer", "minimum": 1},
         "base": {"type": "string"},
         "model": {"type": "string"},
         "context_length": {"type": "integer", "minimum": 1},
@@ -101,6 +102,7 @@ class AgentFinalReport(CoreModel):
     run_id: Optional[uuid.UUID] = None
     run_name: Optional[str] = None
     service_yaml: Optional[str] = None
+    trial: Optional[PositiveInt] = None
     base: Optional[str] = None
     model: Optional[str] = None
     context_length: Optional[PositiveInt] = None
@@ -114,6 +116,7 @@ def validate_report(self) -> Self:
                 "run_id",
                 "run_name",
                 "service_yaml",
+                "trial",
                 "base",
                 "model",
                 "context_length",

src/dstack/_internal/cli/models/presets.py

@@ -67,15 +67,10 @@ class PresetBenchmark(CoreModel):
 
     @property
     def effective_output_tok_per_s(self) -> float:
-        """Performance as defined in the agent prompt's `## Performance`. Derived
-        rather than read, so a miscomputed field cannot become the displayed truth."""
         return self.metrics.total_output_tokens / self.metrics.duration_seconds
 
     @property
     def effective_per_user_tok_per_s(self) -> float:
-        """Per-user output speed as the serving literature defines it: the steady
-        decode rate, `1/TPOT`, which excludes time to first token. Dividing the
-        aggregate by concurrency instead folds TTFT and the ramp into it."""
         return 1000 / self.metrics.tpot_ms.p50
 
     @field_validator("tool", "tool_version", "command")
@@ -113,25 +108,22 @@ def validate_metrics(self) -> Self:
 
 class PresetValidationReplica(CoreModel):
     resources: list[ResourcesSpec]
-    """Exact resources for each running replica in this service replica group."""
 
 
 class PresetValidation(CoreModel):
     replicas: list[PresetValidationReplica]
-    """Ordered to match `ServiceConfiguration.replica_groups`."""
     benchmark: PresetBenchmark
 
 
 class Preset(CoreModel):
     base: str
-    """Base model used for local preset lookup."""
     id: str
     name: Optional[str] = None
-    """Mutable human name; at most one preset or in-flight session holds it."""
     model: str
-    """Exact repo/path loaded by the service command."""
     context_length: PositiveInt
-    """Token context length this preset was verified to support."""
+    trial: Optional[PositiveInt] = None
+    min_context_length: Optional[PositiveInt] = None
+    max_ttft: Optional[PositiveInt] = None
     created_at: datetime
     service: ServiceConfiguration
     validations: list[PresetValidation]

src/dstack/_internal/cli/services/presets/agent.py

@@ -23,6 +23,7 @@
     print_preset_progress,
 )
 from dstack._internal.cli.services.presets.tail import (
+    _DirectoryMirror,
     _FileLineReader,
     _OffsetStore,
     _ProgressTailer,
@@ -171,6 +172,9 @@ def build_preset_agent_env(
     env[_PROGRESS_ENV] = str(workspace.progress_path)
     for name in ["TMPDIR", "TEMP", "TMP"]:
         env[name] = str(workspace.temp_path)
+    # Sandbox the agent's Claude config under the workspace home when we pass our
+    # own API key; under subscription auth keep the real HOME so it reuses the
+    # user's existing `claude` login.
     if auth.api_key is not None:
         env["ANTHROPIC_API_KEY"] = auth.api_key
         env["HOME"] = str(workspace.dstack_home)
@@ -222,14 +226,12 @@ async def run_preset_agent(
             # failure report from the agent returns immediately.
             if output.report_data is not None or output.error is None:
                 return output
-            # A failed attempt that produced agent work is a new outage, not a
-            # continuation of the previous one: restore the full retry budget.
-            # Attempts that fail without any work drain it, so the loop always
-            # terminates when the network stays down.
+            # Only reset the retry budget when the last attempt made progress; a
+            # run that keeps stalling exhausts its retries instead of retrying a
+            # stuck agent forever.
             if output.made_progress:
                 retry_delays = list(_RESUME_DELAYS_SECONDS)
-            # An externally recorded stop is a decision, not an outage: never
-            # resurrect an agent another CLI just terminated.
+            # Another process marked this session interrupted; don't restart it.
             if agent_session.read_manifest().get("status") == "interrupted":
                 return output
             if not retry_delays:
@@ -276,9 +278,9 @@ async def _run_claude_process(
                 stdout=stdout_file,
                 stderr=stderr_file,
                 start_new_session=not IS_WINDOWS,
-                # Inherit only the redirected std handles, not the CLI's other fds.
-                # Without this the untrusted agent inherits our open descriptors, and
-                # on Windows the broad inheritance flakes CreateProcess (WinError 87).
+                # So the untrusted agent inherits only the redirected std handles,
+                # not our other descriptors; broad inheritance also flakes
+                # CreateProcess on Windows (WinError 87).
                 close_fds=True,
             )
         agent_session.update_manifest(
@@ -358,6 +360,8 @@ def _build_claude_command(
 
 
 def _prepare_subprocess_command(command: list[str]) -> list[str]:
+    """On Windows a `.bat`/`.cmd` Claude launcher can't be exec'd directly; wrap
+    it in `cmd.exe /c`. Every other case is returned unchanged."""
     if not IS_WINDOWS or Path(command[0]).suffix.lower() not in {".bat", ".cmd"}:
         return command
     comspec = os.getenv("COMSPEC") or shutil.which("cmd.exe")
@@ -399,7 +403,6 @@ async def _session_tailers(
     redacted_values: Sequence[str],
     offset_store: _OffsetStore,
 ) -> AsyncIterator[None]:
-    """Mirrors the session's progress and record files while the body runs."""
     progress_tailer = _ProgressTailer(
         path=workspace.progress_path,
         redacted_values=redacted_values,
@@ -415,20 +418,16 @@ async def _session_tailers(
             offset_key="runs",
             echo=agent_session.echo,
         ),
-        _RecordMirror(
-            source=workspace.trials_path,
-            target=agent_session.trials_path,
+        _DirectoryMirror(
+            source=workspace.trials_dir,
+            target=agent_session.trials_dir,
             redacted_values=redacted_values,
-            offset_store=offset_store,
-            offset_key="trials",
             echo=agent_session.echo,
         ),
-        _RecordMirror(
-            source=workspace.verifications_path,
-            target=agent_session.verifications_path,
+        _DirectoryMirror(
+            source=workspace.service_dir,
+            target=agent_session.service_dir,
             redacted_values=redacted_values,
-            offset_store=offset_store,
-            offset_key="verifications",
             echo=agent_session.echo,
         ),
     ]
@@ -456,8 +455,7 @@ async def _collect_agent_output(
     is_alive: Callable[[], bool],
     offset_store: _OffsetStore,
 ) -> PresetAgentProcessOutput:
-    """Parses the agent's stream files until it exits; safe alongside a live
-    process or over the remains of a finished one."""
+    """Safe to run alongside a live agent or over the stream files a finished one left behind."""
     stdout_output, _ = await asyncio.gather(
         _read_process_stream(
             stream=_FileLineReader(
@@ -495,8 +493,7 @@ async def attach_preset_agent(
     redacted_values: Sequence[str],
     agent_session: PresetAgentSession,
 ) -> PresetAgentProcessOutput:
-    """Follows a detached session's agent to completion, like
-    `run_preset_agent` without owning the process."""
+    """Like `run_preset_agent`, but tails a detached agent it does not own."""
     offset_store = open_session_offsets(agent_session)
     async with _session_tailers(
         workspace=workspace,
@@ -574,8 +571,7 @@ async def _read_process_stream(
 
 
 async def _terminate_process(proc: asyncio.subprocess.Process) -> None:
-    """SIGTERM, a grace period, then SIGKILL — the same ladder as
-    `terminate_agent_process`, driven through the owned process handle."""
+    """Twin of `terminate_agent_process` for a process this CLI owns, driven through its handle."""
     if IS_WINDOWS:
         await asyncio.to_thread(_terminate_windows_process_tree, proc.pid)
         await proc.wait()
@@ -601,9 +597,7 @@ async def _terminate_process(proc: asyncio.subprocess.Process) -> None:
 
 
 def terminate_agent_process(manifest: dict[str, Any]) -> None:
-    """Terminates the session's agent process tree, if alive. The same
-    SIGTERM-grace-SIGKILL ladder as `_terminate_process`, driven by pid because
-    the caller (`preset stop`) never owned the process."""
+    """Twin of `_terminate_process` driven by pid, because the caller (`preset stop`) never owned the process."""
     agent_pid = manifest.get("agent_pid")
     if not isinstance(agent_pid, int) or not _pid_alive(
         agent_pid, manifest.get("agent_started_at")

src/dstack/_internal/cli/services/presets/apply.py

@@ -11,6 +11,7 @@
     format_preset_objective,
 )
 from dstack._internal.cli.services.presets.store import PresetStore
+from dstack._internal.cli.utils.common import warn
 from dstack._internal.core.errors import CLIError
 from dstack._internal.core.models.configurations import ServiceConfiguration
 from dstack._internal.core.models.profiles import ProfileParams
@@ -54,15 +55,15 @@ def apply_preset(
 
 
 def _validate_preset_matches(preset: Preset, *, configuration: PresetConfiguration) -> None:
-    """The referenced preset must serve what the configuration asks for."""
     model_name = configuration.model.api_model_name
     service_model = preset.service.model
     if service_model is None or service_model.name.lower() != model_name.lower():
         raise CLIError(f"Preset {preset.id} does not serve {model_name}")
     if configuration.min_context_length is not None:
         if preset.context_length < configuration.min_context_length:
-            raise CLIError(
-                f"Preset {preset.id} does not support context length"
+            warn(
+                f"Preset {preset.id} is verified for context length"
+                f" {preset.context_length}, below the requested"
                 f" {configuration.min_context_length}"
             )
     if configuration.model.allows_variant_selection:
@@ -95,7 +96,5 @@ def _format_requested_model(configuration: PresetConfiguration) -> str:
 
 
 def _format_selected_preset(preset: Preset) -> str:
-    # The formatter dims its own keys; wrapping it again would flatten that.
-    # One line, so the objective and the result are joined rather than columned.
     details = f"{format_preset_objective(preset)} {format_preset_benchmark(preset, verbose=True)}"
     return f"{escape(preset.id)} ({details})"