Skip to content
1 change: 1 addition & 0 deletions changelog.d/fixed-pack-agent-follows-hive-gateway.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- A pack agent that an ACMM level adds to a hive whose operator moved their agents onto one inference gateway (e.g. the L2 `reviewer` on a LiteLLM hive) now joins that gateway with its default model instead of the pack's Copilot CLI, and an already-created pack-owned agent is moved there on the next restart. The gateway is the one every operator-picked backend names, when the agents already on it are a majority of the other enabled agents; pack-owned CLI agents such as the L2 `supervisor`/`brainstorm` do not veto the move and follow it, and once moved they count toward that majority so the next level's agent (the L3 `ci-maintainer`) follows too, while a hive with an operator-picked CLI agent or only a minority on a gateway is left as is (#14678).
6 changes: 6 additions & 0 deletions src/crux/config/agent_runtime.go
Original file line number Diff line number Diff line change
Expand Up @@ -647,6 +647,12 @@ func (a AgentConfig) BackendIsOperatorOwned() bool {
return a.BackendOwner == FieldOwnerOperator
}

// BackendIsPackOwned reports whether an ACMM pack wrote this agent's backend,
// as opposed to an operator's pick or a legacy value with no owner.
func (a AgentConfig) BackendIsPackOwned() bool {
return a.BackendOwner == FieldOwnerPack
}

// PauseIsOperatorOwned reports whether an operator explicitly owns this
// agent's pause/run state β€” stamped at dashboard-API create and on an explicit
// operator resume β€” which makes the agent immune to the ACMM pack visibility
Expand Down
7 changes: 7 additions & 0 deletions src/crux/config/leaf_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,13 @@ func TestAgentConfigJevAndOwners(t *testing.T) {
if a.ModelIsOperatorOwned() {
t.Fatal("pack owner is not operator")
}
if a.BackendIsPackOwned() {
t.Fatal("operator-owned backend is not pack-owned")
}
a.BackendOwner = FieldOwnerPack
if !a.BackendIsPackOwned() || (AgentConfig{}).BackendIsPackOwned() {
t.Fatal("only a pack-stamped backend is pack-owned")
}
}

func TestAgentConfigRepos(t *testing.T) {
Expand Down
120 changes: 116 additions & 4 deletions src/pkg/dashboard/api_packs.go
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,9 @@ func (s *Server) applyPack(level int, forceGovernor bool) (*ApplyPackResult, err
// the manager's process table and were re-registered. A repair, not a
// creation β€” it never feeds `created` or isFirstApplyOrExpansion (#5632).
var readded []string
// Read once, before this apply creates or moves anything, so the
// gateway the operator chose is judged on their agents alone.
hiveBackends := s.hiveAgentBackends()
for _, pa := range pack.Agents {
// A deliberately deleted agent is NOT re-created, at any level. The
// pack listing it is exactly why deletion did not stick: ApplyPack
Expand All @@ -156,6 +159,7 @@ func (s *Server) applyPack(level int, forceGovernor bool) (*ApplyPackResult, err
tombstoned = append(tombstoned, pa.Name)
continue
}
hiveGateway := s.hiveGatewayBackend(hiveBackends, pa.Name)
if existing, exists := s.deps.Config.Agents[pa.Name]; exists {
changed := false

Expand Down Expand Up @@ -205,6 +209,19 @@ func (s *Server) applyPack(level int, forceGovernor bool) (*ApplyPackResult, err
// stick. A pack-owned model is cleared there instead, so the
// gateway default applies.
packBackend, backendReplaced := s.packAgentBackend(pa)
// A pack agent still sitting on the CLI the pack gave it follows
// the hive's gateway (#14678). Only a backend the pack itself
// wrote moves β€” never an operator's pick, nor a legacy unowned one.
gw := gatewayPlacement(packBackend, backendReplaced, hiveGateway)
if gw != "" && existing.BackendIsPackOwned() && strings.EqualFold(existing.Backend, packBackend) {
s.logger.Info("moving pack-owned agent onto the hive's inference gateway",
"agent", pa.Name, "backend", existing.Backend, "gateway", gw)
existing.Backend = gw
if !existing.ModelIsOperatorOwned() {
existing.Model = ""
}
changed = true
}
onForeignGateway := s.agentOnForeignGatewayBackend(existing.Backend, packBackend)
packModel := pa.Model
// A hand-set backend other than the pack's (an operator-owned
Expand Down Expand Up @@ -370,6 +387,12 @@ func (s *Server) applyPack(level int, forceGovernor bool) (*ApplyPackResult, err
if backendReplaced {
packModel = ""
}
// The pack model is an id from the pack CLI's catalogue; on the
// gateway it is cleared so the gateway's default_model applies (#9945).
if gw := gatewayPlacement(packBackend, backendReplaced, hiveGateway); gw != "" {
packBackend = gw
packModel = ""
}
agentCfg := config.AgentConfig{
Backend: packBackend,
Model: packModel,
Expand Down Expand Up @@ -1107,10 +1130,6 @@ func (s *Server) ensureCPUTier(level int) {
}
}

// packAgentBackend is the backend a pack agent is placed on: the pack's own
// choice, unless the spoke's backends allow/deny list (#11310) disallows it,
// in which case the list's fallback ("" = hive default) is used instead and
// replaced reports true.
// agentOnForeignGatewayBackend reports whether an agent runs on an
// inference-routed backend (litellm/vllm/llm-d or a configured gateway name)
// other than the pack's backend, so a pack model β€” an id from the pack
Expand All @@ -1122,9 +1141,102 @@ func (s *Server) agentOnForeignGatewayBackend(agentBackend, packBackend string)
return config.IsInferenceBackend(agentBackend) || s.deps.Config.Governor.ResolveGateway(agentBackend) != nil
}

// packAgentBackend is the backend a pack agent is placed on: the pack's own
// choice, unless the spoke's backends allow/deny list (#11310) disallows it,
// in which case the list's fallback ("" = hive default) is used instead and
// replaced reports true.
func (s *Server) packAgentBackend(pa config.PackAgent) (backend string, replaced bool) {
if s.deps.Config.BackendAllowed(pa.Backend) {
return pa.Backend, false
}
return s.deps.Config.Backends.Fallback(), true
}

// agentBackendVote is one enabled agent's backend as seen by
// hiveGatewayBackend: the backend and whether an operator picked it.
type agentBackendVote struct {
backend string
operatorOwned bool
}

// hiveAgentBackends snapshots the backend of every enabled agent, keyed by
// name, for hiveGatewayBackend.
func (s *Server) hiveAgentBackends() map[string]agentBackendVote {
backends := map[string]agentBackendVote{}
for name, agent := range s.deps.Config.Agents {
if !agent.Enabled || s.deps.Config.IsAgentRemoved(name) {
continue
}
backends[name] = agentBackendVote{
backend: strings.TrimSpace(agent.Backend),
operatorOwned: agent.BackendIsOperatorOwned(),
}
}
return backends
}

// hiveGatewayBackend is the inference gateway the operator moved the hive
// onto: the one inference backend or configured gateway that every
// operator-picked backend among the enabled agents other than except names,
// when the agents already on that gateway are a majority of all those other
// agents. It is "" when any operator-picked backend is a subscription CLI,
// the hive default or a different gateway, when there are no operator picks,
// or when the gateway hosts at most half of the other agents. except is the
// pack agent being placed, so its own pack CLI placement is not counted
// against the move.
//
// A pack agent a level adds later is placed on this backend instead of its
// pack CLI (#14678): an operator who moved scanner/quality/guide onto a LiteLLM
// gateway found the L2 `reviewer`, added afterwards, on copilot β€” the only
// agent in the hive still calling an inference provider they had moved off.
// Only operator-owned backends choose the gateway, so the pack's own CLI
// placements (the L2 `supervisor`/`brainstorm`, or a reviewer the pack
// created before this fix) cannot veto the move they are the subject of, and
// the majority bar keeps one agent moved onto a gateway from migrating the
// pack's CLI agents with it. Every agent already on the chosen gateway counts
// toward that majority, whoever placed it there: the pack agents an earlier
// level moved onto it must not dilute the bar for the agent the next level
// adds, or the L3 `ci-maintainer` would land on copilot one level up.
func (s *Server) hiveGatewayBackend(backends map[string]agentBackendVote, except string) string {
shared := ""
for name, vote := range backends {
if strings.EqualFold(name, except) || !vote.operatorOwned {
continue
}
if !s.agentOnForeignGatewayBackend(vote.backend, "") {
return ""
}
if shared != "" && !strings.EqualFold(shared, vote.backend) {
return ""
}
shared = vote.backend
}
if shared == "" || !s.deps.Config.BackendAllowed(shared) {
return ""
}
onGateway, others := 0, 0
for name, vote := range backends {
if strings.EqualFold(name, except) {
continue
}
others++
if strings.EqualFold(vote.backend, shared) {
onGateway++
}
}
if onGateway*2 <= others {
return ""
}
return shared
}

// gatewayPlacement is the backend a pack-owned agent is moved to so it follows
// the hive's gateway (hiveGatewayBackend), or "" to keep packBackend. Only a
// pack CLI placement is redirected: a pack that itself names a gateway, or a
// placement the spoke's backends list already replaced, is left alone.
func gatewayPlacement(packBackend string, backendReplaced bool, hiveGateway string) string {
if hiveGateway == "" || backendReplaced || !config.IsCLIBackend(packBackend) {
return ""
}
return hiveGateway
}
Loading
Loading