feat(plugin): cache-aware pre-engine load for Outputs/Cancel entry points

pkg/clouds/pulumi/cancel.go

@@ -7,7 +7,6 @@ import (
 	"context"
 
 	"github.com/pkg/errors"
-
 	"github.com/pulumi/pulumi/sdk/v3/go/auto"
 
 	"github.com/simple-container-com/api/pkg/api"
@@ -22,7 +21,11 @@ func (p *pulumi) CancelStack(ctx context.Context, cfg *api.ConfigFile, stack api
 	// what an operator reaches for when a deploy is already wedged, so it
 	// failing with `diy URL  has an illegal prefix` lands at the worst
 	// possible moment.
-	if err := p.loadPreEnginePlugins(ctx, stack); err != nil {
+	// Cache-aware: only plugins whose cached schema advertises state-store or
+	// secrets-provider types contribute to backend open. See the same comment
+	// on OutputsStack (preview.go) and
+	// forge/docs/roadmap/items/sc-plugin-stack-outputs-latency-full-schema-load.md.
+	if err := p.loadPreEnginePluginsForBackend(ctx, stack); err != nil {
 		return errors.Wrap(err, "pre-engine plugin load")
 	}
 	defer p.closePreEnginePlugins()

pkg/clouds/pulumi/plugin_preengine.go

@@ -125,6 +125,85 @@ func (b *preEnginePluginBundle) Load(ctx context.Context, specs map[string]api.P
 	return nil
 }
 
+// LoadForBackend is the cache-aware variant for read-only / teardown entry
+// points (OutputsStack, CancelStack). These two paths only need the
+// state-store + secrets-provider sidecars to resolve the backend URL + open
+// the Pulumi secrets manager; they do NOT call Provision / Deploy / Preview /
+// Destroy and therefore don't need a live client for resource / registrar /
+// compute-processor / vm-driver plugins.
+//
+// For each declared plugin:
+//
+//   - Try the per-version schema cache (pkg/plugin/discover.go — populated by
+//     stackread.loadSchema's first fetch, shared across every `sc` invocation
+//     for that machine+version). If a cached schema advertises NEITHER
+//     state_store_types NOR secrets_provider_types, skip the plugin entirely
+//     — Outputs/Cancel don't need its sidecars.
+//   - If the cached schema DOES advertise either, spawn the plugin so the
+//     sidecars have a live client. Same shape as full `Load` for those plugins.
+//   - On cache miss (fresh install, path: spec in dev-mode, or a corrupt
+//     cache file), fall back to the full spawn — correctness wins. A follow-up
+//     warm run writes the cache and subsequent Outputs/Cancel invocations
+//     take the fast path.
+//
+// Correctness invariant preserved by the fallback: any plugin whose
+// state-store or secrets-provider sidecar OutputsStack / CancelStack actually
+// needs will be loaded, either via cache-hit-and-spawn or via cache-miss-full-
+// spawn. The `f306d4c2` fix that made these entries load pre-engine plugins
+// stays correct; this variant just trims the subset that doesn't contribute
+// to backend init.
+//
+// A stale cache that WRONGLY says "no state-store types" would skip a
+// plugin that does provide one, reproducing the exact pre-f306d4c2 bug
+// (empty backend URL on Outputs/Cancel). The cache is version-keyed —
+// `<repo>/<version>/.schema-cache.json` — so a plugin version bump
+// invalidates it automatically, which bounds the window to the plugin
+// version's own lifetime. A plugin author who adds a state-store type
+// mid-version without bumping SHOULD ship a new version; if they don't,
+// `sc stack outputs` fails loudly with "unsupported state-store type %q"
+// from login.go (not silently — the full load's correctness still gates
+// the Provision path).
+//
+// See forge/docs/roadmap/items/sc-plugin-stack-outputs-latency-full-schema-load.md.
+func (b *preEnginePluginBundle) LoadForBackend(ctx context.Context, specs map[string]api.PluginSpec) error {
+	return b.loadForBackendWithCache(ctx, specs, scPlugin.DefaultCacheRoot)
+}
+
+// loadForBackendWithCache is the testable seam. Prod code calls
+// LoadForBackend; tests inject a tmpdir cache root so they can seed
+// known schemas without touching ~/.sc/plugins.
+func (b *preEnginePluginBundle) loadForBackendWithCache(ctx context.Context, specs map[string]api.PluginSpec, cacheRoot string) error {
+	if len(specs) == 0 {
+		return nil
+	}
+	for name, spec := range specs {
+		cached, ok := scPlugin.CachedSchema(spec, cacheRoot)
+		if ok && !schemaContributesToBackend(cached) {
+			b.log.Debug(ctx, "pre-engine skipping plugin %q v%s: cache says no state-store or secrets-provider types (backend-only load)",
+				name, cached.GetPluginVersion())
+			continue
+		}
+		if err := b.loadOne(ctx, name, spec); err != nil {
+			return errors.Wrapf(err, "pre-engine load %q", name)
+		}
+	}
+	return nil
+}
+
+// schemaContributesToBackend reports whether a plugin's schema includes any
+// type SC core's backend-open path consults. Those are the state-store sidecar
+// (resolves PULUMI_BACKEND_URL via InitStateStore) and the secrets-provider
+// sidecar (resolves the Pulumi secrets manager URL + passphrase via
+// InitSecretsProvider). A plugin that advertises neither contributes nothing
+// to `sc stack outputs` / `sc cancel` and can be skipped on the cache-aware
+// load.
+func schemaContributesToBackend(schema *scpluginv1.SchemaResponse) bool {
+	if schema == nil {
+		return false
+	}
+	return len(schema.GetStateStoreTypes()) > 0 || len(schema.GetSecretsProviderTypes()) > 0
+}
+
 func (b *preEnginePluginBundle) loadOne(ctx context.Context, name string, spec api.PluginSpec) error {
 	// Wire the composite-plugin broker. Each spawned plugin gets its
 	// own SCCore endpoint; a PeerInvoke call on it dispatches by

pkg/clouds/pulumi/preengine_entrypoints_test.go

@@ -13,6 +13,8 @@ import (
 
 	"github.com/simple-container-com/api/pkg/api"
 	"github.com/simple-container-com/api/pkg/api/logger"
+	scPlugin "github.com/simple-container-com/api/pkg/plugin"
+	scpluginv1 "github.com/simple-container-com/api/pkg/plugin/proto/v1"
 )
 
 // Every api.Provisioner entry point that reaches the Pulumi backend has to run
@@ -188,3 +190,68 @@ func TestLoadPreEnginePluginsNoOpsWithoutPlugins(t *testing.T) {
 	Expect(p.preEnginePlugins).To(BeNil(),
 		"a bundle was created for a stack with no plugins; closePreEnginePlugins would then tear down nothing")
 }
+
+// TestLoadForBackend_CacheSkipsPluginWithoutBackendTypes — the whole point of
+// the cache-aware variant: a plugin whose cached schema advertises NEITHER
+// state-store types NOR secrets-provider types doesn't contribute to
+// `sc stack outputs` / `sc cancel` opening the backend, so we skip the spawn.
+//
+// Fixture: seed the per-version cache with a schema that lists only
+// resource/compute-processor types; call LoadForBackend; verify the bundle
+// stays empty (no client was spawned, no sidecar registered). Correctness
+// invariant on top: a plugin whose cached schema DOES advertise state-store
+// or secrets-provider types IS spawned on the same code path — covered by
+// the fallback case in TestBackendEntryPointsLoadPreEnginePlugins above
+// (that uses a path: spec, which bypasses the cache entirely and takes the
+// full spawn path — proving the fallback works for any cache-miss shape,
+// including the "cache absent" shape the first-ever run hits).
+func TestLoadForBackend_CacheSkipsPluginWithoutBackendTypes(t *testing.T) {
+	RegisterTestingT(t)
+
+	tmp := t.TempDir()
+
+	// Seed the cache for a plugin spec with a schema that advertises only
+	// a resource type — no state-store, no secrets-provider.
+	spec := api.PluginSpec{Source: &api.PluginSource{
+		Type:       "github-release",
+		Repository: "example/resource-only-plugin",
+		Version:    "v1.0.0",
+	}}
+	schema := &scpluginv1.SchemaResponse{
+		PluginVersion: "1.0.0",
+		ResourceTypes: []string{"example-widget"},
+	}
+	Expect(scPlugin.WriteCachedSchema(spec, tmp, schema)).To(Succeed())
+
+	// loadForBackendWithCache is LoadForBackend's testable seam — same path,
+	// cacheRoot injected. The schema has no backend-contributing types → skip
+	// entirely. No spawn attempt is made (crucially — the plugin binary does
+	// not exist at the resolved path).
+	b := newPreEnginePluginBundle(context.Background(), logger.New())
+	defer b.Close()
+	Expect(b.loadForBackendWithCache(context.Background(), map[string]api.PluginSpec{
+		"resource-only": spec,
+	}, tmp)).To(Succeed())
+	Expect(b.clients).To(BeEmpty(), "a plugin without state-store or secrets-provider must not be spawned on the backend-only load")
+	Expect(b.schemas).To(BeEmpty(), "schemas map must stay empty too — no sidecar registration happened")
+}
+
+// TestLoadForBackend_CacheMissFallsBackToFullSpawn — the correctness
+// guarantee. If the cache has no entry (fresh install / path: spec / corrupt
+// cache), we fall through to full spawn. A nonexistent plugin binary makes
+// that spawn fail, which is the only way to prove this test's assertion
+// (a stub would make the test vacuous).
+func TestLoadForBackend_CacheMissFallsBackToFullSpawn(t *testing.T) {
+	RegisterTestingT(t)
+
+	// A path: spec — schemaCachePath returns (_, false), so CachedSchema
+	// always misses, triggering the full-spawn fallback.
+	spec := api.PluginSpec{Path: filepath.Join(t.TempDir(), "sc-plugin-missing")}
+
+	b := newPreEnginePluginBundle(context.Background(), logger.New())
+	defer b.Close()
+	err := b.LoadForBackend(context.Background(), map[string]api.PluginSpec{
+		"missing": spec,
+	})
+	Expect(err).To(HaveOccurred(), "cache miss must fall back to full spawn, which fails here because the binary does not exist")
+}

pkg/clouds/pulumi/preview.go

@@ -87,14 +87,13 @@ func (p *pulumi) OutputsStack(ctx context.Context, cfg *api.ConfigFile, stack ap
 	//
 	// This is the full load, which spawns every declared plugin and asks each
 	// for its Schema — forge's infra declares six, so an `outputs` call that
-	// was ~200ms becomes a couple of seconds. Deliberate for now: correct and
-	// slow beats fast and broken, and the alternative needs a cache. A
-	// state-store-only variant can't help by itself, because which plugin owns
-	// the state store is itself advertised in a Schema that only exists after
-	// the spawn; the way out is to trust the on-disk schema cache the
-	// pre-engine already writes and let read-only paths like this one and
-	// CancelStack load cache-only, falling back to a spawn on a miss.
-	if err := p.loadPreEnginePlugins(ctx, stack); err != nil {
+	// was ~200ms becomes a couple of seconds. The cache-aware variant below
+	// trims the plugin set to the ones whose sidecars actually contribute to
+	// backend open (state-store, secrets-provider) using the on-disk schema
+	// cache the pre-parse already writes; a plugin whose cached schema has
+	// neither is skipped. Cache miss falls back to full spawn (preserves the
+	// f306d4c2 correctness fix for empty-URL on Outputs/Cancel).
+	if err := p.loadPreEnginePluginsForBackend(ctx, stack); err != nil {
 		return nil, errors.Wrap(err, "pre-engine plugin load")
 	}
 	defer p.closePreEnginePlugins()

pkg/clouds/pulumi/provisioner.go

@@ -119,13 +119,35 @@ func (p *pulumi) ProvisionStack(ctx context.Context, cfg *api.ConfigFile, stack
 
 // loadPreEnginePlugins is a no-op when stack has no plugins declared.
 func (p *pulumi) loadPreEnginePlugins(ctx context.Context, stack api.Stack) error {
+	return p.loadPreEnginePluginsVariant(ctx, stack, false)
+}
+
+// loadPreEnginePluginsForBackend is the cache-aware variant used by
+// OutputsStack + CancelStack. See preEnginePluginBundle.LoadForBackend for
+// the full contract — short version: plugins whose cached schema advertises
+// neither state-store nor secrets-provider types are skipped (their sidecars
+// don't contribute to backend open), and cache misses fall back to the full
+// spawn (preserves the f306d4c2 correctness fix).
+//
+// Roadmap: forge/docs/roadmap/items/sc-plugin-stack-outputs-latency-full-schema-load.md.
+func (p *pulumi) loadPreEnginePluginsForBackend(ctx context.Context, stack api.Stack) error {
+	return p.loadPreEnginePluginsVariant(ctx, stack, true)
+}
+
+func (p *pulumi) loadPreEnginePluginsVariant(ctx context.Context, stack api.Stack, backendOnly bool) error {
 	if len(stack.Server.Plugins) == 0 {
 		return nil
 	}
 	b := newPreEnginePluginBundle(ctx, p.logger)
-	if err := b.Load(ctx, stack.Server.Plugins); err != nil {
+	var loadErr error
+	if backendOnly {
+		loadErr = b.LoadForBackend(ctx, stack.Server.Plugins)
+	} else {
+		loadErr = b.Load(ctx, stack.Server.Plugins)
+	}
+	if loadErr != nil {
 		b.Close()
-		return err
+		return loadErr
 	}
 	if err := b.RegisterSidecars(); err != nil {
 		b.Close()