diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 22535722..ad393f13 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -77,6 +77,15 @@ mkdocs build --strict # must pass with no warnings (CI enforces this) (SWE-bench Docker, gated datasets) must be gated behind a flag and documented. - Match the style and comment density of the surrounding code. +- **Where a new `evolve()` knob goes.** Decision / plug-in behaviour belongs on + a [`Policies`](docs/policies.md) field. Spend, stop, wall-clock, workers, + hooks and recording belong on + `BudgetSettings` / `RuntimeSettings` / `ObservabilitySettings` + (`agentdescent.evolve_settings`). Do **not** add another top-level + `evolve(...)` keyword without updating `EVOLVE_PARAM_ALLOWLIST` and saying + why a bag or Policies field was not enough — the allowlist test fails accidental + growth (#804). Flat kwargs that already exist stay as shortcuts. + ## Pull requests 1. Branch from `main` (agents: a fresh clone per task; nobody pushes to `main` diff --git a/agentdescent/__init__.py b/agentdescent/__init__.py index b82f3e3b..421a93c3 100644 --- a/agentdescent/__init__.py +++ b/agentdescent/__init__.py @@ -154,6 +154,13 @@ list_checkpoints, ) from .budget import BudgetGovernor, CallBudget, budgeted_completion +from .evolve_settings import ( + BudgetSettings, + EVOLVE_PARAM_ALLOWLIST, + EVOLVE_STABLE_SUBSET, + ObservabilitySettings, + RuntimeSettings, +) from .consolidation import ( ConsolidationRecord, ConsolidationStore, @@ -1006,6 +1013,11 @@ "BudgetGovernor", "CallBudget", "budgeted_completion", + "BudgetSettings", + "RuntimeSettings", + "ObservabilitySettings", + "EVOLVE_PARAM_ALLOWLIST", + "EVOLVE_STABLE_SUBSET", # value-directed budget allocation "AllocatorContext", "AllocatorPolicy", diff --git a/agentdescent/async_evolve.py b/agentdescent/async_evolve.py index 416b56e8..9c35067f 100644 --- a/agentdescent/async_evolve.py +++ b/agentdescent/async_evolve.py @@ -176,6 +176,15 @@ def async_evolve( #: aggregator buckets their diffs by target. Empty (the default) is the old #: single-artifact behaviour exactly. extra_artifacts: Optional[Dict[str, "EvolvingArtifact"]] = None, + #: Typed spend/stop bag (#804). Flat ``max_seconds`` / ``max_iters`` / + #: ``max_calls`` / ``max_tokens`` / … stay as shortcuts. ``max_rollouts`` + #: on :class:`~agentdescent.evolve_settings.BudgetSettings` is this + #: function's ``max_iters``. + budget: Optional["BudgetSettings"] = None, + #: Typed runtime bag (#804). Flat kwargs remain shortcuts. + runtime: Optional["RuntimeSettings"] = None, + #: Typed observability bag (#804). Flat kwargs remain shortcuts. + observability: Optional["ObservabilitySettings"] = None, ) -> EvolutionResult: """Evolve an artifact **without a round barrier**. @@ -411,7 +420,7 @@ def async_evolve( rather than being accepted and ignored: a caller who passes a custom acceptance rule and sees a finished run would reasonably conclude it ran. New capabilities go here rather than adding another parameter to a - function that already has thirty-five. + function whose flat surface is already large — prefer a Policies field or a BudgetSettings / RuntimeSettings / ObservabilitySettings bag (#804). allocator: Exactly as in :func:`~agentdescent.evolution.evolve`: value-directed budget allocation (see :mod:`agentdescent.allocator`). ``None`` (the @@ -446,6 +455,18 @@ def async_evolve( strategy, so it is the primary plus every extra), and the aggregator merges their diffs by target. Empty (the default) is the old single-artifact behaviour exactly. + budget: + Typed spend/stop bag + (:class:`~agentdescent.evolve_settings.BudgetSettings`). Flat + ``max_seconds`` / ``max_iters`` / ``max_calls`` / ``max_tokens`` / … + remain shortcuts; the bag field ``max_rollouts`` is this function's + ``max_iters``. + runtime: + Typed runtime bag + (:class:`~agentdescent.evolve_settings.RuntimeSettings`). + observability: + Typed observability bag + (:class:`~agentdescent.evolve_settings.ObservabilitySettings`). Returns ------- @@ -470,6 +491,77 @@ def async_evolve( single merge can contribute two entries (the sweep that ingested its cards, then the one that landed its decision). """ + from .evolve_settings import ( + resolve_budget, resolve_observability, resolve_runtime, + ) + + _budget = resolve_budget( + budget, + max_seconds=max_seconds, max_rollouts=max_iters, max_calls=max_calls, + max_tokens=max_tokens, + stop_on_diminishing_returns=stop_on_diminishing_returns, + efficiency_floor=efficiency_floor, call_budget=call_budget, + target_reward=target_reward, patience=patience, + oracle_budget=oracle_budget, + ) + _runtime = resolve_runtime( + runtime, + n_workers=n_workers, eval_concurrency=eval_concurrency, + max_worker_errors=max_worker_errors, async_ratio=async_ratio, + resync_on_commit=resync_on_commit, pipelined_gate=pipelined_gate, + gate_workers=gate_workers, self_verify=self_verify, + held_out_frac=held_out_frac, checkpointing=checkpointing, + cheap_eval_tasks=cheap_eval_tasks, fusion_tournament=fusion_tournament, + solved_threshold=solved_threshold, shuffle=shuffle, seed=seed, + population_islands=population_islands, + ) + _obs = resolve_observability( + observability, + on_round=on_round, on_reports=on_reports, stop_when=stop_when, + verbose=verbose, usage=usage, + record_trajectories=record_trajectories, + trajectory_store=trajectory_store, + ) + max_seconds = 20.0 if _budget.max_seconds is None else _budget.max_seconds + # async historically defaulted max_seconds to 20.0 when the parameter + # default (also 20.0) was left unset. resolve_budget with bag=None copies + # the flat value, so a caller who passes max_seconds=None explicitly still + # gets None here — keep the historical 20.0 floor for that case too. + if max_seconds is None: + max_seconds = 20.0 + max_iters = _budget.max_rollouts + max_calls = _budget.max_calls + max_tokens = _budget.max_tokens + stop_on_diminishing_returns = _budget.stop_on_diminishing_returns + efficiency_floor = _budget.efficiency_floor + call_budget = _budget.call_budget + target_reward = _budget.target_reward + patience = _budget.patience + oracle_budget = _budget.oracle_budget + n_workers = _runtime.n_workers + eval_concurrency = _runtime.eval_concurrency + max_worker_errors = _runtime.max_worker_errors + async_ratio = _runtime.async_ratio + resync_on_commit = _runtime.resync_on_commit + pipelined_gate = _runtime.pipelined_gate + gate_workers = _runtime.gate_workers + self_verify = _runtime.self_verify + held_out_frac = _runtime.held_out_frac + checkpointing = _runtime.checkpointing + cheap_eval_tasks = _runtime.cheap_eval_tasks + fusion_tournament = _runtime.fusion_tournament + solved_threshold = _runtime.solved_threshold + shuffle = _runtime.shuffle + seed = _runtime.seed + population_islands = _runtime.population_islands + on_round = _obs.on_round + on_reports = _obs.on_reports + stop_when = _obs.stop_when + verbose = _obs.verbose + usage = _obs.usage + record_trajectories = _obs.record_trajectories + trajectory_store = _obs.trajectory_store + _pol = _resolve_policies(policies, "async_evolve()", supported=_ASYNC_WIRED_POLICIES, task_sampler=task_sampler, diff --git a/agentdescent/evolution.py b/agentdescent/evolution.py index db7d9240..e68bd712 100644 --- a/agentdescent/evolution.py +++ b/agentdescent/evolution.py @@ -3065,6 +3065,18 @@ def evolve( #: (``CladeThompson``, ``FlatPuct``) score *clades* -- the metaproductivity #: signal HGM's population search is built on. population_islands: int = 0, + #: Typed spend/stop bag (#804). Prefer this (or ``runtime`` / + #: ``observability``) for new numerics and flags rather than another + #: top-level kwarg. Flat ``max_seconds=`` / ``max_rollouts=`` / … stay as + #: shortcuts onto the bag's fields — see + #: :class:`~agentdescent.evolve_settings.BudgetSettings`. + budget: Optional["BudgetSettings"] = None, + #: Typed runtime bag (#804): workers, gates, split, search extras. Flat + #: ``n_workers=`` / ``rounds=`` / … stay as shortcuts. + runtime: Optional["RuntimeSettings"] = None, + #: Typed observability bag (#804): hooks, meters, trajectory recording. + #: Flat ``on_round=`` / ``verbose=`` / … stay as shortcuts. + observability: Optional["ObservabilitySettings"] = None, ) -> EvolutionResult: """Evolve an artifact. Provide either ``agent`` (with ``solve``/``propose``) or the ``run`` / ``propose`` callables directly. @@ -3391,7 +3403,7 @@ def evolve( rather than being accepted and ignored: a caller who passes a custom acceptance rule and sees a finished run would reasonably conclude it ran. New capabilities go here rather than adding another parameter to a - function that already has thirty-five. + function whose flat surface is already large — prefer a Policies field or a BudgetSettings / RuntimeSettings / ObservabilitySettings bag (#804). allocator: Value-directed budget allocation (see :mod:`agentdescent.allocator`). ``None`` (the default) is the old behaviour exactly: optional spend @@ -3458,6 +3470,18 @@ def evolve( (``CladeThompson``, ``FlatPuct``) score *clades* -- the metaproductivity signal HGM's population search is built on. Islands checkpoint and resume like the rest of the population archive. + budget: + Typed spend/stop bag (:class:`~agentdescent.evolve_settings.BudgetSettings`). + Prefer this for new budget knobs; flat ``max_seconds`` / ``max_rollouts`` + / ``max_calls`` / ``max_tokens`` / … remain shortcuts onto its fields. + See #804 and :data:`~agentdescent.evolve_settings.EVOLVE_STABLE_SUBSET`. + runtime: + Typed runtime bag (:class:`~agentdescent.evolve_settings.RuntimeSettings`). + Workers, gates, data split, search extras. Flat kwargs remain shortcuts. + observability: + Typed observability bag + (:class:`~agentdescent.evolve_settings.ObservabilitySettings`). Hooks, + meters, trajectory recording. Flat kwargs remain shortcuts. Returns ------- @@ -3471,6 +3495,77 @@ def evolve( """ from concurrent.futures import ThreadPoolExecutor, wait as futures_wait from .parallel import DataParallel + from .evolve_settings import ( + resolve_budget, resolve_observability, resolve_runtime, + ) + + # Fold typed bags + flat shortcuts. Bags are the extension point (#804); + # flat kwargs keep working for pip callers and for JSON spec.evolve knobs. + _budget = resolve_budget( + budget, + max_seconds=max_seconds, max_rollouts=max_rollouts, max_calls=max_calls, + max_tokens=max_tokens, + stop_on_diminishing_returns=stop_on_diminishing_returns, + efficiency_floor=efficiency_floor, call_budget=call_budget, + target_reward=target_reward, patience=patience, + oracle_budget=oracle_budget, + ) + _runtime = resolve_runtime( + runtime, + rounds=rounds, n_workers=n_workers, max_concurrency=max_concurrency, + refresh_interval=refresh_interval, round_timeout=round_timeout, + eval_concurrency=eval_concurrency, max_worker_errors=max_worker_errors, + async_ratio=async_ratio, resync_on_commit=resync_on_commit, + pipelined_gate=pipelined_gate, gate_workers=gate_workers, + self_verify=self_verify, held_out_frac=held_out_frac, + checkpointing=checkpointing, cheap_eval_tasks=cheap_eval_tasks, + fusion_tournament=fusion_tournament, solved_threshold=solved_threshold, + shuffle=shuffle, seed=seed, population_islands=population_islands, + ) + _obs = resolve_observability( + observability, + on_round=on_round, on_reports=on_reports, stop_when=stop_when, + verbose=verbose, usage=usage, + record_trajectories=record_trajectories, + trajectory_store=trajectory_store, + ) + max_seconds = _budget.max_seconds + max_rollouts = _budget.max_rollouts + max_calls = _budget.max_calls + max_tokens = _budget.max_tokens + stop_on_diminishing_returns = _budget.stop_on_diminishing_returns + efficiency_floor = _budget.efficiency_floor + call_budget = _budget.call_budget + target_reward = _budget.target_reward + patience = _budget.patience + oracle_budget = _budget.oracle_budget + rounds = _runtime.rounds + n_workers = _runtime.n_workers + max_concurrency = _runtime.max_concurrency + refresh_interval = _runtime.refresh_interval + round_timeout = _runtime.round_timeout + eval_concurrency = _runtime.eval_concurrency + max_worker_errors = _runtime.max_worker_errors + async_ratio = _runtime.async_ratio + resync_on_commit = _runtime.resync_on_commit + pipelined_gate = _runtime.pipelined_gate + gate_workers = _runtime.gate_workers + self_verify = _runtime.self_verify + held_out_frac = _runtime.held_out_frac + checkpointing = _runtime.checkpointing + cheap_eval_tasks = _runtime.cheap_eval_tasks + fusion_tournament = _runtime.fusion_tournament + solved_threshold = _runtime.solved_threshold + shuffle = _runtime.shuffle + seed = _runtime.seed + population_islands = _runtime.population_islands + on_round = _obs.on_round + on_reports = _obs.on_reports + stop_when = _obs.stop_when + verbose = _obs.verbose + usage = _obs.usage + record_trajectories = _obs.record_trajectories + trajectory_store = _obs.trajectory_store if asynchronous: # barrier-free mode: hand the same plug-ins to the async runtime. `rounds` @@ -3547,7 +3642,11 @@ def evolve( record_trajectories=record_trajectories, trajectory_store=trajectory_store, contract=contract, extra_artifacts=extra_artifacts, - population_islands=population_islands) + population_islands=population_islands, + # Bags already folded into the flat locals above; forward them so + # the async surface does not look like it dropped the new knobs, + # and a bag-only caller still works if resolve is re-run (#804). + budget=budget, runtime=runtime, observability=observability) if pipelined_gate: # The mirror of the block above, and the same reasoning: a knob accepted diff --git a/agentdescent/evolve_settings.py b/agentdescent/evolve_settings.py new file mode 100644 index 00000000..e897710a --- /dev/null +++ b/agentdescent/evolve_settings.py @@ -0,0 +1,170 @@ +"""Typed settings bags for :func:`evolve` / :func:`async_evolve`. + +:class:`~agentdescent.policies.Policies` holds replaceable *pieces* +(selection, acceptance, proposal, …). These bags hold the numeric / flag +surface that kept growing on ``evolve()`` after Policies landed (#804). + +**Where a new knob goes** + +* Decision / plug-in behaviour → a :class:`~agentdescent.policies.Policies` + field (or a typed config object that field accepts). +* Spend / stop / wall-clock → :class:`BudgetSettings` (preferred) or a flat + shortcut onto it. +* Workers / gates / data split / search extras → :class:`RuntimeSettings`. +* Hooks / meters / trajectory recording → :class:`ObservabilitySettings`. +* Do **not** add another top-level ``evolve(...)`` keyword without updating + :data:`EVOLVE_PARAM_ALLOWLIST` and the docs — the allowlist test fails a PR + that grows the flat surface by accident. + +Flat kwargs remain shortcuts: they keep working for pip callers and for +JSON ``spec.evolve`` knobs. When a bag is passed, an explicit non-default +flat value still wins (same idea as :meth:`Policies.merged_with`). +""" + +from __future__ import annotations + +from dataclasses import dataclass, fields, replace +from typing import Any, Callable, Dict, Optional, Sequence, TYPE_CHECKING + +if TYPE_CHECKING: + from .agents import Usage + from .budget import CallBudget + from .evolution import RoundInfo + from .trajectory import TrajectoryStore + + +#: Public ``evolve()`` parameter names frozen as of #804. A PR that adds a +#: raw kwarg must update this set on purpose (and prefer a bag / Policies +#: field instead). ``async_evolve`` has its own surface; this freezes the +#: synchronous entry that Spec / MCP / CLI mirror. +EVOLVE_PARAM_ALLOWLIST = frozenset({ + "tasks", "reward", "agent", "run", "propose", "strategy", "parallel", + "task_sampler", "initial_state", "blast_radius", "artifact_id", "rounds", + "n_workers", "max_concurrency", "refresh_interval", "round_timeout", + "target_reward", "patience", "max_worker_errors", "eval_concurrency", + "asynchronous", "async_ratio", "resync_on_commit", "pipelined_gate", + "gate_workers", "max_seconds", "max_rollouts", "max_calls", "max_tokens", + "stop_on_diminishing_returns", "efficiency_floor", "call_budget", + "self_verify", "held_out_frac", "repo_path", "checkpointing", "agg_config", + "staleness_policy", "aggregator_factory", "oracle_budget", + "cheap_eval_tasks", "fusion_tournament", "solved_threshold", "shuffle", + "seed", "on_round", "on_reports", "stop_when", "verbose", "usage", + "policies", "allocator", "record_trajectories", "trajectory_store", + "contract", "extra_artifacts", "population_islands", + # typed bags (#804) — preferred home for new numerics / flags + "budget", "runtime", "observability", +}) + + +#: The short stable contract callers should learn first. Everything else is +#: advanced (Policies, bags, or a documented flat shortcut). +EVOLVE_STABLE_SUBSET = ( + "tasks", "reward", "agent", "run", "propose", "strategy", "policies", + "rounds", "n_workers", "max_concurrency", "max_seconds", "max_rollouts", + "max_calls", "max_tokens", "repo_path", "budget", "runtime", "observability", +) + + + +def _resolve(cls: type, bag: Optional[Any], flat: Dict[str, Any]) -> Any: + base = bag if bag is not None else cls() + overrides: Dict[str, Any] = {} + for f in fields(cls): + if f.name not in flat: + continue + val = flat[f.name] + default = f.default + if bag is None: + overrides[f.name] = val + elif val != default: + overrides[f.name] = val + return replace(base, **overrides) if overrides else base + + +@dataclass(frozen=True) +class BudgetSettings: + """Spend and stop bounds shared by the sync and async drivers. + + Prefer ``evolve(..., budget=BudgetSettings(...))`` (or + ``async_evolve(..., budget=...)``) over adding another top-level kwarg. + Flat ``max_seconds=`` / ``max_rollouts=`` / … remain shortcuts onto these + fields. On the async path, ``max_rollouts`` is the same quantity + ``async_evolve`` historically called ``max_iters``. + """ + + max_seconds: Optional[float] = None + max_rollouts: Optional[int] = None + max_calls: Optional[int] = None + max_tokens: Optional[int] = None + stop_on_diminishing_returns: bool = False + efficiency_floor: float = 0.25 + call_budget: Optional["CallBudget"] = None + target_reward: Optional[float] = None + patience: Optional[int] = None + oracle_budget: int = 200 + + def merged_with(self, **kwargs: Any) -> "BudgetSettings": + """This bag with any explicit keyword taking precedence.""" + return _resolve(BudgetSettings, self, kwargs) + + +@dataclass(frozen=True) +class RuntimeSettings: + """How the loop runs: workers, gates, data split, search extras.""" + + rounds: int = 15 + n_workers: int = 4 + max_concurrency: int = 1 + refresh_interval: int = 1 + round_timeout: Optional[float] = None + eval_concurrency: int = 8 + max_worker_errors: int = 3 + async_ratio: int = 3 + resync_on_commit: bool = True + pipelined_gate: bool = False + gate_workers: int = 2 + self_verify: bool = True + held_out_frac: float = 0.4 + checkpointing: bool = False + cheap_eval_tasks: Optional[int] = None + fusion_tournament: Optional[bool] = None + solved_threshold: float = 0.999 + shuffle: bool = False + seed: int = 0 + population_islands: int = 0 + + def merged_with(self, **kwargs: Any) -> "RuntimeSettings": + return _resolve(RuntimeSettings, self, kwargs) + + +@dataclass(frozen=True) +class ObservabilitySettings: + """Hooks, meters, and durable trajectory recording.""" + + on_round: Optional[Callable[["RoundInfo"], None]] = None + on_reports: Optional[Callable[[Sequence[Any]], None]] = None + stop_when: Optional[Callable[["RoundInfo"], bool]] = None + verbose: bool = False + usage: Optional["Usage"] = None + record_trajectories: bool = False + trajectory_store: Optional["TrajectoryStore"] = None + + def merged_with(self, **kwargs: Any) -> "ObservabilitySettings": + return _resolve(ObservabilitySettings, self, kwargs) + + +def resolve_budget(budget: Optional[BudgetSettings], **flat: Any) -> BudgetSettings: + """Fold flat budget shortcuts into a :class:`BudgetSettings`.""" + return _resolve(BudgetSettings, budget, flat) + + +def resolve_runtime(runtime: Optional[RuntimeSettings], **flat: Any) -> RuntimeSettings: + """Fold flat runtime shortcuts into a :class:`RuntimeSettings`.""" + return _resolve(RuntimeSettings, runtime, flat) + + +def resolve_observability( + observability: Optional[ObservabilitySettings], + **flat: Any) -> ObservabilitySettings: + """Fold flat observability shortcuts into a :class:`ObservabilitySettings`.""" + return _resolve(ObservabilitySettings, observability, flat) diff --git a/agentdescent/policies.py b/agentdescent/policies.py index 5223287e..13fefbf9 100644 --- a/agentdescent/policies.py +++ b/agentdescent/policies.py @@ -491,11 +491,16 @@ def reap(self, older_than: float) -> int: ... class Policies: """Every replaceable piece, in one argument. - `evolve()` has thirty-five parameters. Each capability after this one -- - executors, sandbox providers, cache backends -- would otherwise add another - handful, to two entry points, plus the generated API reference and the - public-surface test. So they go here instead, and the existing keyword - arguments stay as shortcuts onto these fields; nothing is deprecated. + `evolve()` has a large flat parameter surface. Each *behaviour* capability + after this one -- executors, sandbox providers, cache backends -- would + otherwise add another handful, to two entry points, plus the generated API + reference and the public-surface test. So they go here instead, and the + existing keyword arguments stay as shortcuts onto these fields; nothing is + deprecated. Numerics and flags that are not replaceable pieces go in + :class:`~agentdescent.evolve_settings.BudgetSettings` / + :class:`~agentdescent.evolve_settings.RuntimeSettings` / + :class:`~agentdescent.evolve_settings.ObservabilitySettings` (#804), not as + another top-level ``evolve`` kwarg. Every field is ``None`` by default and ``None`` means "the current behaviour", so `Policies()` and passing nothing are the same run. diff --git a/changelog.d/804-evolve-settings-bags.md b/changelog.d/804-evolve-settings-bags.md new file mode 100644 index 00000000..524d3188 --- /dev/null +++ b/changelog.d/804-evolve-settings-bags.md @@ -0,0 +1,2 @@ +Changed +`evolve` / `async_evolve` accept typed `BudgetSettings`, `RuntimeSettings` and `ObservabilitySettings` bags; flat kwargs stay as shortcuts. Docs and `EVOLVE_PARAM_ALLOWLIST` freeze the flat surface so new knobs go in Policies or a bag (#804). diff --git a/docs/algorithm-authoring.md b/docs/algorithm-authoring.md index 92d02d87..3fcc4395 100644 --- a/docs/algorithm-authoring.md +++ b/docs/algorithm-authoring.md @@ -143,3 +143,4 @@ Acceptance for this path: a newcomer can add a named algorithm and see it in * Growing the research catalog itself ([#384](https://github.com/ThinkFlowLab/agentdescent/issues/384)). * Full MethodPolicy / `examples/` microports. * Engine changes (`evolve()`, aggregators, hosts). +* New `evolve()` knobs: Policies field or `BudgetSettings` / `RuntimeSettings` / `ObservabilitySettings` (#804) — not a raw top-level kwarg. diff --git a/docs/api.md b/docs/api.md index 3dba0e8d..a4ea6b38 100644 --- a/docs/api.md +++ b/docs/api.md @@ -13,7 +13,7 @@ means the parameter has none. Each section links to the page that explains *why* the module is shaped the way it is; this page is the *what*. -598 public names across 120 modules. +603 public names across 121 modules. --- @@ -269,7 +269,10 @@ evolve( trajectory_store: Optional['TrajectoryStore'] = None, contract: Optional[Contract] = None, extra_artifacts: Optional[Dict[str, 'EvolvingArtifact']] = None, - population_islands: int = 0 + population_islands: int = 0, + budget: Optional['BudgetSettings'] = None, + runtime: Optional['RuntimeSettings'] = None, + observability: Optional['ObservabilitySettings'] = None ) -> EvolutionResult ``` @@ -325,13 +328,16 @@ evolve( | `stop_when` | `Optional[Callable[['RoundInfo'], bool]]` | `None` | Called after `on_round` with the same `RoundInfo`; return `True` to end the run with `stop_reason="stop_when"`. This is the seam for a budget the engine does not know how to count -- dollars from a shared `Usage`, an external deadline, a kill file. It is asked where `max_seconds` / `max_calls` are, so it stops between rounds and never mid-merge, and the run keeps what it has committed. An exception raised here is reported, not fatal. | | `verbose` | `bool` | `False` | Print a line per round. Independent of the `RuntimeWarning` emitted when a run ends early -- that always fires. | | `usage` | `Optional[Usage]` | `None` | Share one `Usage` with your model adapters (`claude(usage=u)`, `openai_compatible(usage=u)`) and the result's token counts become real. Without it the run still reports calls, seconds and failures -- `run` is `(rendered, task) -> str`, so an opaque actor has no way to surface tokens, and inventing a number would be worse than reporting zero. | -| `policies` | `Optional['Policies']` | `None` | Bundle of replaceable pieces (`Policies`). Every field defaults to `None` meaning "current behaviour", so `Policies()` and passing nothing are the same run. The individual keyword arguments -- `task_sampler`, `staleness_policy`, `aggregator_factory` -- are shortcuts onto its fields and keep working; an explicit argument wins over a bundle default rather than being silently ignored. Fields whose implementations have not landed yet raise rather than being accepted and ignored: a caller who passes a custom acceptance rule and sees a finished run would reasonably conclude it ran. New capabilities go here rather than adding another parameter to a function that already has thirty-five. | +| `policies` | `Optional['Policies']` | `None` | Bundle of replaceable pieces (`Policies`). Every field defaults to `None` meaning "current behaviour", so `Policies()` and passing nothing are the same run. The individual keyword arguments -- `task_sampler`, `staleness_policy`, `aggregator_factory` -- are shortcuts onto its fields and keep working; an explicit argument wins over a bundle default rather than being silently ignored. Fields whose implementations have not landed yet raise rather than being accepted and ignored: a caller who passes a custom acceptance rule and sees a finished run would reasonably conclude it ran. New capabilities go here rather than adding another parameter to a function whose flat surface is already large — prefer a Policies field or a BudgetSettings / RuntimeSettings / ObservabilitySettings bag (#804). | | `allocator` | `Optional[Any]` | `None` | Value-directed budget allocation (see `allocator`). `None` (the default) is the old behaviour exactly: optional spend (fusion tournaments, the self-verify rollout) degrades against the global `soft_floor` / `hard_floor` thresholds as the token budget runs out, every candidate treated alike. Pass a `ValueBudgetAllocator` (or any object with `decide(ctx) -> AllocateDecision` and `observe(ctx, committed)`) and, once the budget is tight, the self-verify rollout is spent only on candidates the allocator's value model expects to commit; the merge path reports every outcome back so the model learns. Requires `max_tokens` to be set (the allocator decides how the budget is spent; `max_tokens` is what exists to spend). Counterfactual exploration re-measures candidates the model judged unworthy, so a biased value model is corrected rather than reinforced. | | `record_trajectories` | `bool` | `False` | Persist every rollout and gate evaluation as a `Trajectory` under `/trajectories/` -- what the run provider saw, what it returned, what that earned, against which artifact version and environment. Off by default: a throwaway repo's records die with it, so the write cost is pure overhead there, and recording is a deliberate choice about keeping a run's evidence. The store is the durable base every learning-type increment reads -- process-supervised reflection, offline re-weighting, audit replay -- and it is what an `EvidenceCard.trace_ids` entry points at. Pass `trajectory_store=` with a store you built to control where records land instead. | | `trajectory_store` | `Optional['TrajectoryStore']` | `None` | An explicit `TrajectoryStore` to record into. Pass one to control where records land (a shared store, a custom root, a custom privacy configuration) without going through the `/trajectories/` default. The explicit store wins over `record_trajectories`; `None` + `record_trajectories` builds the default store. | | `contract` | `Optional[Contract]` | `None` | The artifact's contract -- the externally-visible interface, and where governance invariants are declared. Pass `Contract(invariants=(Invariant("mutable_keys", ("a", "b")),))` and the aggregator enforces each rule **before** a candidate is measured (see `invariants`); a diff that breaks one reports a `violates-invariant` outcome and never buys a held-out sweep. `None` (the default) is the reference skill contract, which declares nothing and enforces nothing -- the untouched path exactly. | | `extra_artifacts` | `Optional[Dict[str, 'EvolvingArtifact']]` | `None` | Additional artifacts to register alongside the primary (`artifact_id`), as `{aid: EvolvingArtifact}`. The primary is the one the worker loop rolls out and proposes on; each extra is registered in the ledger and the worker proposes against it too, so a proposal can name one of them as a diff target and the aggregator merges it there (the evidence buffer already buckets by `diff.target`). `depends_on` on an extra's `Contract` declares which artifacts it relies on: a contract-breaking change to one re-measures its dependents (their cached scores were taken under the superseded contract) and lands atomically via `Ledger.commit_atomic`. Empty (the default) is the old single-artifact behaviour exactly. | | `population_islands` | `int` | `0` | How many **persistent head-islands** the population explores in parallel, on top of `dev` (see `PopulationAggregator`). Requires a declared selection policy (`Policies(selection=...)`). `0` (the default) keeps the old behaviour exactly: per-step forks that reset a policy's lineage. Above zero, `island_count` forks survive across steps and each keeps its own lineage, so tree-shaped policies (`CladeThompson`, `FlatPuct`) score *clades* -- the metaproductivity signal HGM's population search is built on. Islands checkpoint and resume like the rest of the population archive. | +| `budget` | `Optional['BudgetSettings']` | `None` | Typed spend/stop bag (`BudgetSettings`). Prefer this for new budget knobs; flat `max_seconds` / `max_rollouts` / `max_calls` / `max_tokens` / … remain shortcuts onto its fields. See #804 and `EVOLVE_STABLE_SUBSET`. | +| `runtime` | `Optional['RuntimeSettings']` | `None` | Typed runtime bag (`RuntimeSettings`). Workers, gates, data split, search extras. Flat kwargs remain shortcuts. | +| `observability` | `Optional['ObservabilitySettings']` | `None` | Typed observability bag (`ObservabilitySettings`). Hooks, meters, trajectory recording. Flat kwargs remain shortcuts. | ### `reflector(...)` @@ -4533,7 +4539,10 @@ async_evolve( trajectory_store: Optional[Any] = None, contract: Optional[Contract] = None, population_islands: int = 0, - extra_artifacts: Optional[Dict[str, 'EvolvingArtifact']] = None + extra_artifacts: Optional[Dict[str, 'EvolvingArtifact']] = None, + budget: Optional['BudgetSettings'] = None, + runtime: Optional['RuntimeSettings'] = None, + observability: Optional['ObservabilitySettings'] = None ) -> EvolutionResult ``` @@ -4588,13 +4597,90 @@ async_evolve( | `stop_when` | `Optional[Callable[[RoundInfo], bool]]` | `None` | Asked after `on_round` with the same `RoundInfo`; `True` ends the run with `stop_reason="stop_when"` -- the caller's own budget (dollars, a deadline, a kill file), checked between merger sweeps like the built-in bounds. Same thread and the same rules as `on_round`. | | `verbose` | `bool` | `False` | Print one line per merger sweep. | | `usage` | `Optional[Usage]` | `None` | Share one `Usage` with your model adapters (`claude(usage=u)`, `openai_compatible(usage=u)`) and the result's token counts become real. Without it the run still reports calls, seconds and failures -- `run` is `(rendered, task) -> str`, so an opaque actor has no way to surface tokens, and inventing a number would be worse than reporting zero. | -| `policies` | `Optional['Policies']` | `None` | Bundle of replaceable pieces (`Policies`). Every field defaults to `None` meaning "current behaviour", so `Policies()` and passing nothing are the same run. The individual keyword arguments -- `task_sampler`, `staleness_policy`, `aggregator_factory` -- are shortcuts onto its fields and keep working; an explicit argument wins over a bundle default rather than being silently ignored. Fields whose implementations have not landed yet raise rather than being accepted and ignored: a caller who passes a custom acceptance rule and sees a finished run would reasonably conclude it ran. New capabilities go here rather than adding another parameter to a function that already has thirty-five. | +| `policies` | `Optional['Policies']` | `None` | Bundle of replaceable pieces (`Policies`). Every field defaults to `None` meaning "current behaviour", so `Policies()` and passing nothing are the same run. The individual keyword arguments -- `task_sampler`, `staleness_policy`, `aggregator_factory` -- are shortcuts onto its fields and keep working; an explicit argument wins over a bundle default rather than being silently ignored. Fields whose implementations have not landed yet raise rather than being accepted and ignored: a caller who passes a custom acceptance rule and sees a finished run would reasonably conclude it ran. New capabilities go here rather than adding another parameter to a function whose flat surface is already large — prefer a Policies field or a BudgetSettings / RuntimeSettings / ObservabilitySettings bag (#804). | | `allocator` | `Optional[Any]` | `None` | Exactly as in `evolve`: value-directed budget allocation (see `allocator`). `None` (the default) is the old behaviour exactly. Pass a value allocator and, once the token budget is tight, the self-verify rollout is spent only on candidates the allocator expects to commit; the merger reports every outcome back so it learns. Requires `max_tokens`. Counterfactual exploration corrects a biased value model rather than reinforcing it. | | `record_trajectories` | `bool` | `False` | Exactly as in `evolve`: persist every rollout and gate evaluation as a `Trajectory` under `/trajectories/`. Off by default; pass `trajectory_store=` to control where records land. | | `trajectory_store` | `Optional[Any]` | `None` | An explicit `TrajectoryStore` to record into. `None` + `record_trajectories` builds the default store at `/trajectories`. | | `contract` | `Optional[Contract]` | `None` | Exactly as in `evolve`: the artifact's contract, where governance invariants are declared. `None` (the default) is the reference skill contract, which declares nothing and enforces nothing. | | `population_islands` | `int` | `0` | Persistent head-islands for a population run. Exactly as in `evolve`; `0` (the default) keeps the per-step-fork behaviour. | | `extra_artifacts` | `Optional[Dict[str, 'EvolvingArtifact']]` | `None` | Exactly as in `evolve`: additional artifacts to register alongside the primary, as `{aid: EvolvingArtifact}`. The barrier-free worker proposes against each of them too (no parallel strategy, so it is the primary plus every extra), and the aggregator merges their diffs by target. Empty (the default) is the old single-artifact behaviour exactly. | +| `budget` | `Optional['BudgetSettings']` | `None` | Typed spend/stop bag (`BudgetSettings`). Flat `max_seconds` / `max_iters` / `max_calls` / `max_tokens` / … remain shortcuts; the bag field `max_rollouts` is this function's `max_iters`. | +| `runtime` | `Optional['RuntimeSettings']` | `None` | Typed runtime bag (`RuntimeSettings`). | +| `observability` | `Optional['ObservabilitySettings']` | `None` | Typed observability bag (`ObservabilitySettings`). | + +--- + +## Evolve settings bags + +Budget / runtime / observability bags that freeze the flat evolve() surface.  ·  `agentdescent.evolve_settings`  ·  [guide](evolution.md) + +### `BudgetSettings(...)` + +Spend and stop bounds shared by the sync and async drivers. + +```python +BudgetSettings( + max_seconds: Optional[float] = None, + max_rollouts: Optional[int] = None, + max_calls: Optional[int] = None, + max_tokens: Optional[int] = None, + stop_on_diminishing_returns: bool = False, + efficiency_floor: float = 0.25, + call_budget: Optional['CallBudget'] = None, + target_reward: Optional[float] = None, + patience: Optional[int] = None, + oracle_budget: int = 200 +) -> None +``` + +| method | what it does | +|---|---| +| `merged_with(**kwargs: Any) -> 'BudgetSettings'` | This bag with any explicit keyword taking precedence. | + +### `ObservabilitySettings(...)` + +Hooks, meters, and durable trajectory recording. + +```python +ObservabilitySettings( + on_round: Optional[Callable[['RoundInfo'], None]] = None, + on_reports: Optional[Callable[[Sequence[Any]], None]] = None, + stop_when: Optional[Callable[['RoundInfo'], bool]] = None, + verbose: bool = False, + usage: Optional['Usage'] = None, + record_trajectories: bool = False, + trajectory_store: Optional['TrajectoryStore'] = None +) -> None +``` + +### `RuntimeSettings(...)` + +How the loop runs: workers, gates, data split, search extras. + +```python +RuntimeSettings( + rounds: int = 15, + n_workers: int = 4, + max_concurrency: int = 1, + refresh_interval: int = 1, + round_timeout: Optional[float] = None, + eval_concurrency: int = 8, + max_worker_errors: int = 3, + async_ratio: int = 3, + resync_on_commit: bool = True, + pipelined_gate: bool = False, + gate_workers: int = 2, + self_verify: bool = True, + held_out_frac: float = 0.4, + checkpointing: bool = False, + cheap_eval_tasks: Optional[int] = None, + fusion_tournament: Optional[bool] = None, + solved_threshold: float = 0.999, + shuffle: bool = False, + seed: int = 0, + population_islands: int = 0 +) -> None +``` --- @@ -5148,6 +5234,14 @@ One run's failure mode, the evidence, and what to try next. The multi-file proposal format a `FileTree` reflector is told to emit. +### `EVOLVE_PARAM_ALLOWLIST` + +Frozen set of public ``evolve()`` parameter names (#804). A PR that adds a raw kwarg must update this on purpose; prefer Policies or a settings bag. + +### `EVOLVE_STABLE_SUBSET` + +Short stable ``evolve()`` contract to teach first; the long tail is advanced (Policies, bags, or a documented flat shortcut). + ### `EloRating` A minimal Elo rating over pairwise matches (K-factor 32). diff --git a/docs/evolution.md b/docs/evolution.md index 36429ba9..c4e143ef 100644 --- a/docs/evolution.md +++ b/docs/evolution.md @@ -24,6 +24,26 @@ print(result.final_reward) # held-out reward That's the minimum. Everything below is optional and swappable. +## Stable subset vs the long tail + +`evolve()` still accepts a large flat keyword surface for backward compatibility +and for JSON `spec.evolve` knobs, but the **stable contract** to learn first is +small: `tasks` / `reward` / `agent` (or `run`+`propose`) / `strategy` / +`policies` / `rounds` (or budgets) / `n_workers` / `repo_path`, plus the typed +bags when you need more knobs together. + +| Prefer | For | +|---|---| +| `Policies(...)` | Replaceable decision pieces (selection, acceptance, proposal, …) | +| `budget=BudgetSettings(...)` | Spend and stop bounds (`max_seconds`, `max_rollouts`, `max_tokens`, …) | +| `runtime=RuntimeSettings(...)` | Workers, gates, data split, search extras | +| `observability=ObservabilitySettings(...)` | Hooks, meters, trajectory recording | + +New capabilities go in one of those four places — not as another top-level +`evolve(...)` keyword (#804). The frozen allowlist is +`EVOLVE_PARAM_ALLOWLIST`; see also [call any algorithm](algorithms.md) / #384 +(kind filter and card vs port fidelity should not require teaching every knob). + !!! tip "Three companion pages" [Module map](modules.md) is the index of every module and a reading order; [API reference](api.md) has every signature, generated from the diff --git a/tests/test_evolve_settings.py b/tests/test_evolve_settings.py new file mode 100644 index 00000000..86a04cf0 --- /dev/null +++ b/tests/test_evolve_settings.py @@ -0,0 +1,123 @@ +"""Typed evolve settings bags and the flat-signature freeze (#804).""" +from __future__ import annotations + +import inspect +import warnings + +import pytest + +from agentdescent import ( + BudgetSettings, + EVOLVE_PARAM_ALLOWLIST, + EVOLVE_STABLE_SUBSET, + ObservabilitySettings, + RuntimeSettings, + Task, + async_evolve, + evolve, +) +from agentdescent.evolution import SingleSlot +from agentdescent.evolve_settings import ( + resolve_budget, + resolve_observability, + resolve_runtime, +) + + +TASKS = [Task(id=str(i), prompt="p", meta={"gold": "x"}) for i in range(8)] + + +def _run(**kw): + return evolve( + TASKS, lambda t, o: 0.5, + run=lambda a, t: "x", propose=lambda *a: None, + strategy=SingleSlot(initial_value="v"), + n_workers=1, rounds=2, **kw, + ) + + +def test_evolve_param_allowlist_matches_signature(): + """A new raw evolve() kwarg must update EVOLVE_PARAM_ALLOWLIST on purpose.""" + names = set(inspect.signature(evolve).parameters) + assert names == set(EVOLVE_PARAM_ALLOWLIST), ( + f"evolve() signature drifted from EVOLVE_PARAM_ALLOWLIST. " + f"added={sorted(names - EVOLVE_PARAM_ALLOWLIST)}; " + f"removed={sorted(EVOLVE_PARAM_ALLOWLIST - names)}. " + "Prefer a Policies field or BudgetSettings / RuntimeSettings / " + "ObservabilitySettings (#804) instead of growing the flat surface." + ) + + +def test_stable_subset_is_subset_of_allowlist(): + missing = [n for n in EVOLVE_STABLE_SUBSET if n not in EVOLVE_PARAM_ALLOWLIST] + assert missing == [] + + +def test_budget_bag_overrides_defaults_and_flat_shortcut_wins(): + bag = BudgetSettings(max_tokens=1000, patience=2) + folded = resolve_budget(bag, max_tokens=None, patience=None, + max_seconds=None, max_rollouts=None, max_calls=None, + stop_on_diminishing_returns=False, + efficiency_floor=0.25, call_budget=None, + target_reward=None, oracle_budget=200) + assert folded.max_tokens == 1000 and folded.patience == 2 + folded2 = resolve_budget(bag, max_tokens=500, patience=None, + max_seconds=None, max_rollouts=None, max_calls=None, + stop_on_diminishing_returns=False, + efficiency_floor=0.25, call_budget=None, + target_reward=None, oracle_budget=200) + assert folded2.max_tokens == 500 and folded2.patience == 2 + + +def test_runtime_and_observability_fold(): + r = resolve_runtime(RuntimeSettings(n_workers=8, rounds=3), + n_workers=4, rounds=15, max_concurrency=1, + refresh_interval=1, round_timeout=None, + eval_concurrency=8, max_worker_errors=3, + async_ratio=3, resync_on_commit=True, + pipelined_gate=False, gate_workers=2, + self_verify=True, held_out_frac=0.4, + checkpointing=False, cheap_eval_tasks=None, + fusion_tournament=None, solved_threshold=0.999, + shuffle=False, seed=0, population_islands=0) + assert r.n_workers == 8 and r.rounds == 3 + seen = [] + obs = resolve_observability( + ObservabilitySettings(verbose=True, on_round=lambda info: seen.append(1)), + on_round=None, on_reports=None, stop_when=None, verbose=False, + usage=None, record_trajectories=False, trajectory_store=None, + ) + assert obs.verbose is True and obs.on_round is not None + + +def test_evolve_honours_budget_and_runtime_bags(): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + res = _run(budget=BudgetSettings(patience=1), + runtime=RuntimeSettings(rounds=6, n_workers=1)) + assert res.error is None + assert res.stop_reason == "patience" + assert len(res.history) <= 3 + + +def test_async_evolve_honours_budget_max_rollouts(): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + res = async_evolve( + TASKS, lambda t, o: 0.5, + run=lambda a, t: "x", propose=lambda *a: None, + strategy=SingleSlot(initial_value="v"), + budget=BudgetSettings(max_rollouts=4), + n_workers=1, max_seconds=10.0, self_verify=False, + ) + assert res.error is None + assert res.stop_reason in ("max_iters", "max_rollouts") + assert res.rollouts <= 5 + + +def test_flat_kwargs_still_work_without_bags(): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + res = _run(patience=1) + assert res.error is None + assert res.stop_reason == "patience" diff --git a/tools/gen_api_docs.py b/tools/gen_api_docs.py index 51cd7491..b51889fa 100644 --- a/tools/gen_api_docs.py +++ b/tools/gen_api_docs.py @@ -213,6 +213,9 @@ ("agentdescent.async_evolve", "Barrier-free evolution", "`evolve()` without the round barrier.", "async.md"), + ("agentdescent.evolve_settings", "Evolve settings bags", + "Budget / runtime / observability bags that freeze the flat evolve() surface.", + "evolution.md"), ("agentdescent.async_runtime", "The async orchestrator", "The reference barrier-free runtime and its statistics.", "async.md"), ("agentdescent.orchestrator", "The reference orchestrator", @@ -222,6 +225,12 @@ #: Symbols exported as plain values (not classes or functions) need a written #: line each -- ``inspect`` has nothing useful to say about an int or a frozenset. CONSTANTS: Dict[str, str] = { + "EVOLVE_PARAM_ALLOWLIST": "Frozen set of public ``evolve()`` parameter names " + "(#804). A PR that adds a raw kwarg must update this " + "on purpose; prefer Policies or a settings bag.", + "EVOLVE_STABLE_SUBSET": "Short stable ``evolve()`` contract to teach first; " + "the long tail is advanced (Policies, bags, or a " + "documented flat shortcut).", "SOLVED": "Reward at or above which a task counts as solved (`0.999`). " "Lower it for a graded scorer, or every rollout asks the reflector " "to fix an answer that was already good.",