diff --git a/cmd/deploy.go b/cmd/deploy.go index f4e3286..df625b9 100644 --- a/cmd/deploy.go +++ b/cmd/deploy.go @@ -7,14 +7,18 @@ import ( "fmt" "math/big" "os" + "strings" "time" "dario.cat/mergo" + "github.com/google/go-containerregistry/pkg/name" "github.com/spf13/cobra" "github.com/spf13/pflag" "github.com/stackrox/roxie/internal/clusterdefaults" "github.com/stackrox/roxie/internal/component" + "github.com/stackrox/roxie/internal/constants" "github.com/stackrox/roxie/internal/deployer" + "github.com/stackrox/roxie/internal/dockerauth" "github.com/stackrox/roxie/internal/env" "github.com/stackrox/roxie/internal/helpers" "github.com/stackrox/roxie/internal/imagetag" @@ -264,7 +268,7 @@ func runDeploy(cmd *cobra.Command, args []string) error { return err } - if err := deployValidate(log, components, &deploySettings); err != nil { + if err := deployValidate(ctx, log, components, &deploySettings); err != nil { return err } @@ -446,7 +450,22 @@ func configureConfig(log *logger.Logger, components component.Component, deployS return nil } -func deployValidate(log *logger.Logger, components component.Component, deploySettings *deployer.Config) error { +// validateImageRegistry checks that registry is a well-formed "host/repository-path" string, e.g. "quay.io/rhacs-eng". +func validateImageRegistry(registry string) error { + host, repoPath, hasPath := strings.Cut(registry, "/") + if !hasPath || repoPath == "" { + return fmt.Errorf("roxie.imageRegistry must include a repository path (e.g. %s), got: %s", constants.DefaultRegistry, registry) + } + if _, err := name.NewRegistry(host); err != nil { + return fmt.Errorf("roxie.imageRegistry has an invalid registry host %q: %w", host, err) + } + if _, err := name.NewRepository(repoPath); err != nil { + return fmt.Errorf("roxie.imageRegistry has an invalid repository path %q: %w", repoPath, err) + } + return nil +} + +func deployValidate(ctx context.Context, log *logger.Logger, components component.Component, deploySettings *deployer.Config) error { if components.IncludesCentral() && os.Getenv("ROXIE_SHELL") != "" { return errors.New("already in a roxie sub-shell (ROXIE_SHELL environment variable is set), please exit the shell and try again") } @@ -455,6 +474,21 @@ func deployValidate(log *logger.Logger, components component.Component, deploySe return errors.New("running without a controlling terminal requires --envrc to be set") } + registry := deploySettings.Roxie.Registry() + if deploySettings.Roxie.UsesCustomRegistry() { + if err := validateImageRegistry(registry); err != nil { + return err + } + + requiresAuth := dockerauth.New(log).RegistryRequiresAuth(ctx, registry) + deploySettings.Roxie.RegistryRequiresAuth = requiresAuth + if requiresAuth { + log.Dimf("Registry %s requires authentication", registry) + } else { + log.Dimf("Registry %s is public, no authentication required", registry) + } + } + clusterType := deploySettings.Roxie.ClusterType if env.RunningInRoxieContainer { @@ -466,10 +500,9 @@ func deployValidate(log *logger.Logger, components component.Component, deploySe return errors.New("containerized mode requires Central exposure") } - // On infra OpenShift we already get image pull secrets for Quay automatically. - if clusterType.NeedsPullSecrets() { + if deploySettings.Roxie.NeedsPullSecrets() { if os.Getenv("REGISTRY_USERNAME") == "" || os.Getenv("REGISTRY_PASSWORD") == "" { - return fmt.Errorf("containerized mode requires REGISTRY_USERNAME and REGISTRY_PASSWORD environment variables for clusters of type %s", clusterType) + return fmt.Errorf("containerized mode requires REGISTRY_USERNAME and REGISTRY_PASSWORD environment variables for registry %s on clusters of type %s", registry, clusterType) } if _, err := os.Stat("/kubeconfig"); err != nil { return fmt.Errorf("containerized mode requires /kubeconfig file: %w", err) @@ -485,6 +518,9 @@ func deployValidate(log *logger.Logger, components component.Component, deploySe if deploySettings.Operator.DeployViaOlmEnabled() { return errors.New("using Konflux images while deploying operator via OLM is not supported") } + if registry != constants.DefaultRegistry { + return fmt.Errorf("using Konflux images with a custom image registry (%s) is not supported", registry) + } } if deploySettings.HasMixedVersions() { diff --git a/cmd/deploy_test.go b/cmd/deploy_test.go index be7976b..a8199d8 100644 --- a/cmd/deploy_test.go +++ b/cmd/deploy_test.go @@ -8,6 +8,7 @@ import ( "time" "dario.cat/mergo" + "github.com/stackrox/roxie/internal/constants" "github.com/stackrox/roxie/internal/deployer" "github.com/stackrox/roxie/internal/imagetag" "github.com/stackrox/roxie/internal/logger" @@ -314,6 +315,55 @@ func TestNewDeployCmd_SetRejectsSpec(t *testing.T) { } } +func TestValidateImageRegistry(t *testing.T) { + tests := []struct { + name string + registry string + expectError bool + errorContains string + }{ + {name: "default registry", registry: constants.DefaultRegistry}, + {name: "valid host/path registry", registry: "quay.io/stackrox-io"}, + {name: "registry host with port", registry: "localhost:5000/rhacs-eng"}, + { + name: "bare host with no path is rejected", + registry: "justahost", + expectError: true, + errorContains: "must include a repository path", + }, + { + name: "trailing slash with no path is rejected", + registry: "quay.io/", + expectError: true, + errorContains: "must include a repository path", + }, + { + name: "invalid registry host", + registry: "quay io/rhacs-eng", + expectError: true, + errorContains: "invalid registry host", + }, + { + name: "invalid repository path characters", + registry: "quay.io/RHACS-ENG", + expectError: true, + errorContains: "invalid repository path", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := validateImageRegistry(tt.registry) + if tt.expectError { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.errorContains) + return + } + require.NoError(t, err) + }) + } +} + func TestApplyUserDefaults(t *testing.T) { log := logger.New() diff --git a/internal/deployer/acs_images.go b/internal/deployer/acs_images.go index 78b8580..de27c91 100644 --- a/internal/deployer/acs_images.go +++ b/internal/deployer/acs_images.go @@ -2,13 +2,11 @@ package deployer import ( "fmt" - - "github.com/stackrox/roxie/internal/constants" ) func imagesForConfig(config Config) []string { var images []string - imageRegistry := constants.DefaultRegistry + imageRegistry := config.Roxie.Registry() for _, instance := range config.OperatorInstances() { prefix := "" @@ -20,8 +18,8 @@ func imagesForConfig(config Config) []string { fmt.Sprintf("%s/%s%s:%s", imageRegistry, prefix, "central-db", instance.Version), fmt.Sprintf("%s/%s%s:%s", imageRegistry, prefix, "scanner-v4-db", instance.Version), fmt.Sprintf("%s/%s%s:%s", imageRegistry, prefix, "scanner-v4", instance.Version), - instance.OperatorImage(), - instance.BundleImage(), + instance.OperatorImage(imageRegistry), + instance.BundleImage(imageRegistry), ) } diff --git a/internal/deployer/addons.go b/internal/deployer/addons.go index 3d683db..2dd47f7 100644 --- a/internal/deployer/addons.go +++ b/internal/deployer/addons.go @@ -31,7 +31,7 @@ func (d *Deployer) deployAddOns(ctx context.Context, addOns []AddOn) error { return nil } - needPullSecrets := d.config.Roxie.ClusterType.NeedsPullSecrets() + needPullSecrets := d.config.Roxie.NeedsPullSecrets() if err := d.prepareNamespace(ctx, d.config.Central.Namespace, needPullSecrets); err != nil { return fmt.Errorf("failed to prepare namespace: %w", err) } diff --git a/internal/deployer/config.go b/internal/deployer/config.go index a8d5df2..e4c3e49 100644 --- a/internal/deployer/config.go +++ b/internal/deployer/config.go @@ -2,6 +2,7 @@ package deployer import ( "fmt" + "strings" "time" "github.com/stackrox/roxie/internal/constants" @@ -56,10 +57,38 @@ func (c *Config) DeepCopy() (*Config, error) { // RoxieConfig holds roxie-level settings such as version and feature flags. type RoxieConfig struct { Version imagetag.MainTag `yaml:"version,omitempty"` + ImageRegistry string `yaml:"imageRegistry,omitempty"` KonfluxImages *bool `yaml:"konfluxImages,omitempty"` FeatureFlags map[string]bool `yaml:"featureFlags,omitempty"` ClusterType types.ClusterType `yaml:"clusterType,omitempty"` HAProxy HAProxyConfig `yaml:"haProxy,omitempty"` + + // RegistryRequiresAuth is computed internally and is not user-configurable. + RegistryRequiresAuth bool `yaml:"-"` +} + +// Registry returns the resolved image registry, defaulting to +// constants.DefaultRegistry when ImageRegistry is not set. +func (c *RoxieConfig) Registry() string { + if c.ImageRegistry == "" { + return constants.DefaultRegistry + } + return strings.TrimSuffix(c.ImageRegistry, "/") +} + +// UsesCustomRegistry returns whether a custom image registry was configured. +func (c *RoxieConfig) UsesCustomRegistry() bool { + return c.Registry() != constants.DefaultRegistry +} + +// NeedsPullSecrets returns whether roxie needs to set up image pull secrets itself. +// For a custom registry this relies on RegistryRequiresAuth having already been +// resolved during deploy validation (see cmd/deploy.go's deployValidate). +func (c *RoxieConfig) NeedsPullSecrets() bool { + if c.UsesCustomRegistry() { + return c.RegistryRequiresAuth + } + return c.ClusterType.NeedsDefaultRegistryPullSecrets() } func (c *RoxieConfig) KonfluxImagesSet() bool { @@ -118,8 +147,7 @@ func (c *OperatorInstanceConfig) ClusterRoleBindingName() string { } // BundleImage returns the operator bundle image for this operator instance. -func (c *OperatorInstanceConfig) BundleImage() string { - imageRegistry := constants.DefaultRegistry +func (c *OperatorInstanceConfig) BundleImage(imageRegistry string) string { operatorTag := c.Version.ToOperatorTag() if c.KonfluxImagesEnabled() { return fmt.Sprintf("%s/release-operator-bundle:v%s", imageRegistry, operatorTag) @@ -127,8 +155,8 @@ func (c *OperatorInstanceConfig) BundleImage() string { return fmt.Sprintf("%s/stackrox-operator-bundle:v%s", imageRegistry, operatorTag) } -func (c *OperatorInstanceConfig) OperatorImage() string { - imageRegistry := constants.DefaultRegistry +// OperatorImage returns the operator image for this operator instance. +func (c *OperatorInstanceConfig) OperatorImage(imageRegistry string) string { operatorTag := c.Version.ToOperatorTag() if c.KonfluxImagesEnabled() { return fmt.Sprintf("%s/release-operator:%s", imageRegistry, operatorTag) diff --git a/internal/deployer/deploy_via_operator.go b/internal/deployer/deploy_via_operator.go index 33b7af7..1ac9785 100644 --- a/internal/deployer/deploy_via_operator.go +++ b/internal/deployer/deploy_via_operator.go @@ -12,13 +12,14 @@ import ( "strings" "time" + "gopkg.in/yaml.v3" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "github.com/stackrox/roxie/internal/component" "github.com/stackrox/roxie/internal/env" "github.com/stackrox/roxie/internal/helpers" "github.com/stackrox/roxie/internal/k8s" "github.com/stackrox/roxie/internal/types" - "gopkg.in/yaml.v3" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) var ( @@ -214,7 +215,7 @@ func (d *Deployer) ensureOperatorDeployedOLM(ctx context.Context) error { func (d *Deployer) deployCentralOperator(ctx context.Context) error { d.logger.Info("🚀 Deploying Central via Operator...") - needPullSecrets := d.config.Roxie.ClusterType.NeedsPullSecrets() + needPullSecrets := d.config.Roxie.NeedsPullSecrets() if err := d.prepareNamespace(ctx, d.config.Central.Namespace, needPullSecrets); err != nil { return fmt.Errorf("failed to prepare namespace: %w", err) } @@ -247,7 +248,8 @@ func (d *Deployer) deployCentralOperator(ctx context.Context) error { return d.configureCentralEndpoint(ctx) } -// isOperatorVersionCorrect checks if the deployed operator matches the desired version. +// isOperatorVersionCorrect checks if the deployed operator matches the desired +// image, comparing the full reference (registry, repository, and tag). func (d *Deployer) isOperatorVersionCorrect(ctx context.Context, instance OperatorInstanceConfig) bool { currentImage, err := d.getDeployedOperatorImage(ctx, instance.Namespace) if err != nil { @@ -255,19 +257,11 @@ func (d *Deployer) isOperatorVersionCorrect(ctx context.Context, instance Operat return false } - // Extract the tag from the current image - parts := strings.SplitN(currentImage, ":", 2) - if len(parts) < 2 { - d.logger.Warningf("Could not parse operator image tag from: %s", currentImage) - return false - } - currentTag := parts[1] - - desiredTag := instance.Version.ToOperatorTag().String() - if currentTag != desiredTag { - d.logger.Info("Operator version mismatch detected:") - d.logger.Infof(" Current: %s", currentTag) - d.logger.Infof(" Desired: %s", desiredTag) + desiredImage := instance.OperatorImage(d.config.Roxie.Registry()) + if currentImage != desiredImage { + d.logger.Info("Operator image mismatch detected:") + d.logger.Infof(" Current: %s", currentImage) + d.logger.Infof(" Desired: %s", desiredImage) return false } return true @@ -309,7 +303,7 @@ func (d *Deployer) ensurePullSecretExists(ctx context.Context, namespace string) return errors.New("no pull secrets available to set up on the cluster") } - pullSecretYAML := d.dockerAuth.CreatePullSecretYAMLFromCredentials(*d.dockerCreds, namespace) + pullSecretYAML := d.dockerAuth.CreatePullSecretYAMLFromCredentials(*d.dockerCreds, namespace, d.config.Roxie.Registry()) _, err := d.runKubectl(ctx, k8s.KubectlOptions{ Args: []string{"apply", "-f", "-"}, Stdin: strings.NewReader(pullSecretYAML), @@ -828,7 +822,7 @@ func (d *Deployer) configureCentralEndpoint(ctx context.Context) error { func (d *Deployer) deploySecuredClusterOperator(ctx context.Context) error { d.logger.Info("🚀 Deploying SecuredCluster via Operator...") - needPullSecrets := d.config.Roxie.ClusterType.NeedsPullSecrets() + needPullSecrets := d.config.Roxie.NeedsPullSecrets() if err := d.prepareNamespace(ctx, d.config.SecuredCluster.Namespace, needPullSecrets); err != nil { return fmt.Errorf("failed to prepare namespace: %w", err) } diff --git a/internal/deployer/deployer.go b/internal/deployer/deployer.go index f399a3c..b829f71 100644 --- a/internal/deployer/deployer.go +++ b/internal/deployer/deployer.go @@ -303,9 +303,9 @@ func (d *Deployer) stopDetachedPortForward() { // Deploy deploys the specified components to the cluster. func (d *Deployer) Deploy(ctx context.Context, components component.Component) error { // Prepare and verify credentials early to fail fast. - needPullSecrets := d.config.Roxie.ClusterType.NeedsPullSecrets() + needPullSecrets := d.config.Roxie.NeedsPullSecrets() if needPullSecrets { - if err := d.prepareCredentials(); err != nil { + if err := d.prepareCredentials(ctx); err != nil { return fmt.Errorf("failed to prepare credentials: %w", err) } } @@ -354,11 +354,11 @@ func (d *Deployer) Deploy(ctx context.Context, components component.Component) e // prepareCredentials prepares and verifies Docker credentials early to allow failing fast. // The verified credentials are stored in the Deployer object for later use. -func (d *Deployer) prepareCredentials() error { +func (d *Deployer) prepareCredentials(ctx context.Context) error { d.logger.Dimf("Preparing and verifying Docker credentials...") // This will retrieve and verify credentials, returning error if invalid - creds, err := d.dockerAuth.GetAndVerifyCredentials() + creds, err := d.dockerAuth.GetAndVerifyCredentials(ctx, d.config.Roxie.Registry()) if err != nil { return err } diff --git a/internal/deployer/konflux_test.go b/internal/deployer/konflux_test.go index 08132e3..6255634 100644 --- a/internal/deployer/konflux_test.go +++ b/internal/deployer/konflux_test.go @@ -12,13 +12,18 @@ import ( func TestOperatorImage_Konflux(t *testing.T) { instance := OperatorInstanceConfig{Version: "4.9.2", KonfluxImages: new(true)} expected := fmt.Sprintf("%s/release-operator:4.9.2", constants.DefaultRegistry) - assert.Equal(t, expected, instance.OperatorImage()) + assert.Equal(t, expected, instance.OperatorImage(constants.DefaultRegistry)) } func TestOperatorImage_NonKonflux(t *testing.T) { instance := OperatorInstanceConfig{Version: "4.9.2", KonfluxImages: new(false)} expected := fmt.Sprintf("%s/stackrox-operator:4.9.2", constants.DefaultRegistry) - assert.Equal(t, expected, instance.OperatorImage()) + assert.Equal(t, expected, instance.OperatorImage(constants.DefaultRegistry)) +} + +func TestOperatorImage_RegistryOverride(t *testing.T) { + instance := OperatorInstanceConfig{Version: "4.9.2"} + assert.Equal(t, "quay.io/stackrox-io/stackrox-operator:4.9.2", instance.OperatorImage("quay.io/stackrox-io")) } func TestPopulateKonfluxEnvVars_AllEntries(t *testing.T) { diff --git a/internal/deployer/operator.go b/internal/deployer/operator.go index 20f5ed0..8f563c5 100644 --- a/internal/deployer/operator.go +++ b/internal/deployer/operator.go @@ -5,14 +5,17 @@ import ( "context" "errors" "fmt" + "net/http" "os" "path/filepath" "strings" "time" + "github.com/google/go-containerregistry/pkg/v1/remote/transport" "gopkg.in/yaml.v3" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "github.com/stackrox/roxie/internal/constants" "github.com/stackrox/roxie/internal/k8s" "github.com/stackrox/roxie/internal/ocihelper" ) @@ -35,7 +38,10 @@ var requiredCRDs = []string{ // deployOperatorNonOLM deploys one RHACS operator instance without OLM. func (d *Deployer) deployOperatorNonOLM(ctx context.Context, instance OperatorInstanceConfig) error { d.logger.Infof("Operator tag: %s (namespace %s)", instance.Version, instance.Namespace) - bundleImage := instance.BundleImage() + bundleImage, err := d.resolveBundleImage(ctx, instance, d.config.Roxie.Registry()) + if err != nil { + return fmt.Errorf("resolving operator bundle image: %w", err) + } bundleDir, err := d.downloadAndExtractOperatorBundle(ctx, bundleImage) if err != nil { @@ -168,7 +174,10 @@ func (d *Deployer) ensureCRDsInstalled(ctx context.Context) error { if len(missing) > 0 { crdInstance := d.config.NewestOperatorInstance() - bundleImage := crdInstance.BundleImage() + bundleImage, err := d.resolveBundleImage(ctx, crdInstance, d.config.Roxie.Registry()) + if err != nil { + return fmt.Errorf("resolving operator bundle image: %w", err) + } d.logger.Warningf("Missing CRDs detected (%s)", strings.Join(missing, ", ")) d.logger.Warningf("Fetching bundle %s", bundleImage) @@ -189,6 +198,37 @@ func (d *Deployer) ensureCRDsInstalled(ctx context.Context) error { return nil } +// resolveBundleImage returns the operator bundle image to use for the given instance, probing +// the configured registry first and falling back to constants.DefaultRegistry if the bundle +// does not exist there. +// +// This is done because upstream StackRox builds (quay.io/stackrox-io) do not publish operator bundles. +func (d *Deployer) resolveBundleImage(ctx context.Context, instance OperatorInstanceConfig, registry string) (string, error) { + bundleImage := instance.BundleImage(registry) + if registry == constants.DefaultRegistry { + return bundleImage, nil + } + + if err := ocihelper.VerifyImageExistence(ctx, d.logger, bundleImage); err != nil { + var te *transport.Error + if errors.As(err, &te) && te.StatusCode == http.StatusNotFound { + fallbackImage := instance.BundleImage(constants.DefaultRegistry) + d.logger.Infof("No operator bundle found at %s, falling back to %s", bundleImage, fallbackImage) + return fallbackImage, nil + } + return "", fmt.Errorf("verifying operator bundle %s: %w", bundleImage, err) + } + + return bundleImage, nil +} + +func needsOperatorPullSecrets(instance OperatorInstanceConfig, roxieConfig *RoxieConfig) bool { + if roxieConfig.UsesCustomRegistry() { + return roxieConfig.RegistryRequiresAuth + } + return instance.KonfluxImagesEnabled() && roxieConfig.ClusterType.NeedsDefaultRegistryPullSecrets() +} + // deployOperatorFromCSV deploys the operator from CSV into the given instance namespace. func (d *Deployer) deployOperatorFromCSV(ctx context.Context, bundleDir string, instance OperatorInstanceConfig) error { csvFile := filepath.Join(bundleDir, "rhacs-operator.clusterserviceversion.yaml") @@ -204,7 +244,7 @@ func (d *Deployer) deployOperatorFromCSV(ctx context.Context, bundleDir string, } serviceAccountName := deploymentSpec["service_account"].(string) - d.useOperatorPullSecrets = instance.KonfluxImagesEnabled() && d.config.Roxie.ClusterType.NeedsPullSecrets() + d.useOperatorPullSecrets = needsOperatorPullSecrets(instance, &d.config.Roxie) d.logger.Info("📋 Operator deployment plan:") d.logger.Dimf(" • Namespace: %s", instance.Namespace) @@ -440,11 +480,11 @@ func (d *Deployer) createDeploymentFromCSV(ctx context.Context, instance Operato return fmt.Errorf("extracting manager container from operator pod spec: %w", err) } + operatorImage := instance.OperatorImage(d.config.Roxie.Registry()) podSpec["serviceAccountName"] = deploymentSpec["service_account"] - if current, _ := managerContainer["image"].(string); current != instance.OperatorImage() { - // Currently this should only happen in Konflux mode. - d.logger.Infof("Rewriting operator image to %s", instance.OperatorImage()) - managerContainer["image"] = instance.OperatorImage() + if current, _ := managerContainer["image"].(string); current != operatorImage { + d.logger.Infof("Rewriting operator image to %s", operatorImage) + managerContainer["image"] = operatorImage } if len(instance.EnvVars) > 0 { diff --git a/internal/deployer/operator_integration_test.go b/internal/deployer/operator_integration_test.go new file mode 100644 index 0000000..f594d0c --- /dev/null +++ b/internal/deployer/operator_integration_test.go @@ -0,0 +1,39 @@ +//go:build integration + +package deployer + +import ( + "context" + "testing" + "time" + + "github.com/stackrox/roxie/internal/constants" + "github.com/stackrox/roxie/internal/logger" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestResolveBundleImage_StackroxIOFallsBackToDefault_Integration(t *testing.T) { + d := &Deployer{logger: logger.New()} + ctx, cancel := context.WithTimeout(t.Context(), 2*time.Minute) + defer cancel() + + instance := OperatorInstanceConfig{Version: "4.11.1"} + + // We don't build operator bundles for upstream StackRox builds, so this should fall back to the rhacs-eng-hosted bundle. + bundleImage, err := d.resolveBundleImage(ctx, instance, "quay.io/stackrox-io") + require.NoError(t, err) + assert.Equal(t, constants.DefaultRegistry+"/stackrox-operator-bundle:v4.11.1", bundleImage, + "should fall back to the rhacs-eng-hosted bundle") +} + +func TestResolveBundleImage_NonNotFoundErrorPropagates_Integration(t *testing.T) { + d := &Deployer{logger: logger.New()} + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) + defer cancel() + + instance := OperatorInstanceConfig{Version: "4.11.1"} + + _, err := d.resolveBundleImage(ctx, instance, "roxie-test-nonexistent-host.invalid/rhacs-eng") + require.Error(t, err) +} diff --git a/internal/deployer/operator_test.go b/internal/deployer/operator_test.go new file mode 100644 index 0000000..12a5990 --- /dev/null +++ b/internal/deployer/operator_test.go @@ -0,0 +1,95 @@ +package deployer + +import ( + "testing" + + "github.com/stackrox/roxie/internal/types" + "github.com/stretchr/testify/assert" +) + +func TestNeedsOperatorPullSecrets(t *testing.T) { + tests := []struct { + name string + instance OperatorInstanceConfig + roxieConfig RoxieConfig + expected bool + }{ + { + name: "default registry, non-Konflux: no pull secrets", + instance: OperatorInstanceConfig{}, + roxieConfig: RoxieConfig{ClusterType: types.ClusterTypeGKE}, + expected: false, + }, + { + name: "Konflux images: pull secrets needed", + instance: OperatorInstanceConfig{KonfluxImages: new(true)}, + roxieConfig: RoxieConfig{ClusterType: types.ClusterTypeGKE}, + expected: true, + }, + { + name: "Konflux images on a cluster type that auto-configures default-registry credentials: no pull secrets", + instance: OperatorInstanceConfig{KonfluxImages: new(true)}, + roxieConfig: RoxieConfig{ClusterType: types.ClusterTypeInfraOpenShift4}, + expected: false, + }, + { + name: "private custom registry, non-Konflux: pull secrets needed", + instance: OperatorInstanceConfig{}, + roxieConfig: RoxieConfig{ImageRegistry: "quay.io/stackrox-io", ClusterType: types.ClusterTypeGKE, RegistryRequiresAuth: true}, + expected: true, + }, + { + name: "private custom registry is never auto-configured, even on a cluster type that auto-configures the default registry", + instance: OperatorInstanceConfig{}, + roxieConfig: RoxieConfig{ImageRegistry: "quay.io/stackrox-io", ClusterType: types.ClusterTypeInfraOpenShift4, RegistryRequiresAuth: true}, + expected: true, + }, + { + name: "public custom registry: no pull secrets needed", + instance: OperatorInstanceConfig{}, + roxieConfig: RoxieConfig{ImageRegistry: "quay.io/stackrox-io", ClusterType: types.ClusterTypeGKE, RegistryRequiresAuth: false}, + expected: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, needsOperatorPullSecrets(tt.instance, &tt.roxieConfig)) + }) + } +} + +func TestRoxieConfig_NeedsPullSecrets(t *testing.T) { + tests := []struct { + name string + roxie RoxieConfig + expected bool + }{ + { + name: "default registry on a cluster type that auto-configures credentials", + roxie: RoxieConfig{ClusterType: types.ClusterTypeInfraOpenShift4}, + expected: false, + }, + { + name: "default registry on a cluster type that doesn't auto-configure credentials", + roxie: RoxieConfig{ClusterType: types.ClusterTypeGKE}, + expected: true, + }, + { + name: "private custom registry, even on a cluster type that auto-configures default-registry credentials", + roxie: RoxieConfig{ImageRegistry: "quay.io/stackrox-io", ClusterType: types.ClusterTypeInfraOpenShift4, RegistryRequiresAuth: true}, + expected: true, + }, + { + name: "public custom registry: no pull secrets needed", + roxie: RoxieConfig{ImageRegistry: "quay.io/stackrox-io", ClusterType: types.ClusterTypeInfraOpenShift4, RegistryRequiresAuth: false}, + expected: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, tt.roxie.NeedsPullSecrets()) + }) + } +} diff --git a/internal/dockerauth/dockerauth.go b/internal/dockerauth/dockerauth.go index fde6bd0..ac7c786 100644 --- a/internal/dockerauth/dockerauth.go +++ b/internal/dockerauth/dockerauth.go @@ -2,22 +2,31 @@ package dockerauth import ( "bytes" + "context" "encoding/base64" "encoding/json" "errors" "fmt" + "net/http" "os" "os/exec" "path/filepath" + "strings" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote/transport" "github.com/stackrox/roxie/internal/constants" "github.com/stackrox/roxie/internal/logger" ) -const ( - acsImageRegistry = "quay.io" - mainImageRepository = "rhacs-eng/main" -) +// splitRegistryHost splits a resolved image registry (e.g. "quay.io/stackrox-io") +// into its host ("quay.io") and org/repo path ("stackrox-io"). +func splitRegistryHost(registry string) (host, path string) { + host, path, _ = strings.Cut(registry, "/") + return host, path +} // DockerAuth handles Docker authentication and pull secret management. type DockerAuth struct { @@ -58,7 +67,10 @@ func New(log *logger.Logger) *DockerAuth { // GetAndVerifyCredentials retrieves and verifies Docker credentials. // This should be called early to fail fast if credentials are invalid. -func (d *DockerAuth) GetAndVerifyCredentials() (*Credentials, error) { +func (d *DockerAuth) GetAndVerifyCredentials(ctx context.Context, registry string) (*Credentials, error) { + host, orgPath := splitRegistryHost(registry) + mainImageRepository := orgPath + "/main" + var username, password string // Try environment variables first. @@ -78,7 +90,7 @@ func (d *DockerAuth) GetAndVerifyCredentials() (*Credentials, error) { d.logger.Dimf("REGISTRY_USERNAME/REGISTRY_PASSWORD unset. Trying to obtain Docker credentials from config file: %s", dockerConfigPath) if _, err := os.Stat(dockerConfigPath); err == nil { var err error - username, password, err = d.getCredentialsFromDockerConfig(dockerConfigPath) + username, password, err = d.getCredentialsFromDockerConfig(dockerConfigPath, host) if err != nil { return nil, err } @@ -91,7 +103,7 @@ func (d *DockerAuth) GetAndVerifyCredentials() (*Credentials, error) { // Verify credentials. if !d.skipCredVerification { - if err := d.VerifyCredentials(username, password); err != nil { + if err := d.VerifyCredentials(ctx, username, password, host, mainImageRepository); err != nil { return nil, fmt.Errorf("credentials are invalid: %w", err) } } @@ -102,8 +114,9 @@ func (d *DockerAuth) GetAndVerifyCredentials() (*Credentials, error) { }, nil } -// getCredentialsFromDockerConfig extracts credentials from existing Docker config. -func (d *DockerAuth) getCredentialsFromDockerConfig(configPath string) (string, string, error) { +// getCredentialsFromDockerConfig extracts credentials from existing Docker config +// for the given registry host. +func (d *DockerAuth) getCredentialsFromDockerConfig(configPath, host string) (string, string, error) { data, err := os.ReadFile(configPath) if err != nil { return "", "", fmt.Errorf("failed to read Docker config: %w", err) @@ -114,8 +127,8 @@ func (d *DockerAuth) getCredentialsFromDockerConfig(configPath string) (string, return "", "", fmt.Errorf("failed to parse Docker config: %w", err) } - // Check for existing auths for the ACS image registry. - if authEntry, ok := config.Auths[acsImageRegistry]; ok && authEntry.Auth != "" { + // Check for existing auths for the target registry host. + if authEntry, ok := config.Auths[host]; ok && authEntry.Auth != "" { // Decode the base64 auth string to get username:password decoded, err := base64.StdEncoding.DecodeString(authEntry.Auth) if err != nil { @@ -128,15 +141,15 @@ func (d *DockerAuth) getCredentialsFromDockerConfig(configPath string) (string, return string(parts[0]), string(parts[1]), nil } - // Try credential helper specifically configured for the ACS image registry - helper := d.lookupCredentialHelperForRegistry(&config, acsImageRegistry) + // Try credential helper specifically configured for the target registry host. + helper := d.lookupCredentialHelperForRegistry(&config, host) if helper == "" { - return "", "", fmt.Errorf("no Docker credentials found in config for ACS image registry (%s)", acsImageRegistry) + return "", "", fmt.Errorf("no Docker credentials found in config for image registry (%s)", host) } - credData, err := d.getCredentialFromHelper(helper, acsImageRegistry) + credData, err := d.getCredentialFromHelper(helper, host) if err != nil { - return "", "", fmt.Errorf("failed to get credentials from helper '%s' for '%s': %w", helper, acsImageRegistry, err) + return "", "", fmt.Errorf("failed to get credentials from helper '%s' for '%s': %w", helper, host, err) } return credData.Username, credData.Secret, nil @@ -177,54 +190,71 @@ func (d *DockerAuth) getCredentialFromHelper(helperName, registry string) (*Cred return &credData, nil } -// VerifyCredentials attempts to verify that the credentials work by making a request to the registry. -// This uses a read-only HTTP request. -// It mimics what the kubelet would do when pulling images. -func (d *DockerAuth) VerifyCredentials(username, password string) error { - // Create auth header for Basic authentication - authString := fmt.Sprintf("%s:%s", username, password) - encodedAuth := base64.StdEncoding.EncodeToString([]byte(authString)) +// VerifyCredentials verifies that the given credentials grant pull access to +// the given repository on the given registry host. It works for registries +// that follow the standard OCI Distribution v2 challenge/token protocol. +func (d *DockerAuth) VerifyCredentials(ctx context.Context, username, password, host, repository string) error { + reg, err := name.NewRegistry(host) + if err != nil { + return fmt.Errorf("invalid registry host %q: %w", host, err) + } - // Try to get a token from quay.io's OAuth2 endpoint for a specific repository - // This mimics what kubelet does when pulling images - it requests a token with pull scope - // for the specific repository. - authURL := fmt.Sprintf("https://%s/v2/auth?service=%s&scope=repository:%s:pull", - acsImageRegistry, acsImageRegistry, mainImageRepository) + auth := &authn.Basic{Username: username, Password: password} + scope := fmt.Sprintf("repository:%s:pull", repository) - cmd := exec.Command("curl", "-s", "-f", - "-H", fmt.Sprintf("Authorization: Basic %s", encodedAuth), - authURL) + if _, err := transport.NewWithContext(ctx, reg, auth, http.DefaultTransport, []string{scope}); err != nil { + return fmt.Errorf("credential verification failed for %s: %w", host, err) + } - output, err := cmd.CombinedOutput() + d.logger.Dimf("Successfully verified credentials for %s (repository: %s)", host, repository) + return nil +} + +// RegistryRequiresAuth makes a best-effort check for whether registry requires +// authentication to pull images, by sending a single anonymous tags-list +// request against a well-known repository path. Anything short of a confirmed +// successful response fails safe by reporting that auth is required. +func (d *DockerAuth) RegistryRequiresAuth(ctx context.Context, registry string) bool { + host, orgPath := splitRegistryHost(registry) + + reg, err := name.NewRegistry(host) if err != nil { - d.logger.Warningf("Failed to verify credentials for %s: %v", acsImageRegistry, err) - d.logger.Dimf("Verification output: %s", string(output)) - return fmt.Errorf("credential verification failed for %s: %w", acsImageRegistry, err) + return true } + repo := reg.Repo(orgPath, "main") - // Check if we got a valid JSON response with a token - var tokenResponse map[string]interface{} - if err := json.Unmarshal(output, &tokenResponse); err != nil { - return fmt.Errorf("credential verification failed: invalid response from %s: %w", acsImageRegistry, err) + tr, err := transport.NewWithContext(ctx, reg, authn.Anonymous, http.DefaultTransport, []string{repo.Scope("pull")}) + if err != nil { + return true } - if _, ok := tokenResponse["token"]; !ok { - return fmt.Errorf("credential verification failed: no token received from %s", acsImageRegistry) + url := fmt.Sprintf("%s://%s/v2/%s/tags/list?n=1", repo.Scheme(), repo.RegistryStr(), repo.RepositoryStr()) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return true } - d.logger.Dimf("Successfully verified credentials for %s (repository: %s)", acsImageRegistry, mainImageRepository) - return nil + resp, err := (&http.Client{Transport: tr}).Do(req) + if err != nil { + return true + } + defer resp.Body.Close() + + return resp.StatusCode < 200 || resp.StatusCode >= 300 } -// CreatePullSecretYAMLFromCredentials creates Kubernetes pull secret YAML from verified credentials. -func (d *DockerAuth) CreatePullSecretYAMLFromCredentials(creds Credentials, namespace string) string { +// CreatePullSecretYAMLFromCredentials creates Kubernetes pull secret YAML from +// verified credentials, scoped to the host of the given image registry +func (d *DockerAuth) CreatePullSecretYAMLFromCredentials(creds Credentials, namespace, registry string) string { + host, _ := splitRegistryHost(registry) + // Create auth string authString := fmt.Sprintf("%s:%s", creds.Username, creds.Password) encodedAuth := base64.StdEncoding.EncodeToString([]byte(authString)) dockerConfig := DockerConfig{ Auths: map[string]AuthEntry{ - acsImageRegistry: {Auth: encodedAuth}, + host: {Auth: encodedAuth}, }, } diff --git a/internal/dockerauth/dockerauth_test.go b/internal/dockerauth/dockerauth_test.go index 597acc2..f9c21ea 100644 --- a/internal/dockerauth/dockerauth_test.go +++ b/internal/dockerauth/dockerauth_test.go @@ -1,12 +1,18 @@ package dockerauth import ( + "context" "encoding/base64" "encoding/json" + "fmt" + "net/http" + "net/http/httptest" "strings" "testing" + "github.com/stackrox/roxie/internal/constants" "github.com/stackrox/roxie/internal/logger" + "github.com/stretchr/testify/assert" ) func TestGetAndVerifyCredentialsFromEnv(t *testing.T) { @@ -18,7 +24,7 @@ func TestGetAndVerifyCredentialsFromEnv(t *testing.T) { da := New(log) da.skipCredVerification = true // Skip verification in tests - creds, err := da.GetAndVerifyCredentials() + creds, err := da.GetAndVerifyCredentials(context.Background(), constants.DefaultRegistry) if err != nil { t.Fatalf("GetAndVerifyCredentials failed: %v", err) } @@ -31,7 +37,7 @@ func TestGetAndVerifyCredentialsFromEnv(t *testing.T) { } // Test creating YAML from credentials - yamlText := da.CreatePullSecretYAMLFromCredentials(*creds, "ns") + yamlText := da.CreatePullSecretYAMLFromCredentials(*creds, "ns", "registry.example.com/some-org") // Verify YAML structure if !strings.Contains(yamlText, "apiVersion: v1") { @@ -72,8 +78,12 @@ func TestGetAndVerifyCredentialsFromEnv(t *testing.T) { t.Fatalf("Decoded data is not valid JSON: %v", err) } - if _, ok := data["auths"]; !ok { - t.Error("Decoded JSON should contain 'auths' key") + auths, ok := data["auths"].(map[string]interface{}) + if !ok { + t.Fatal("Decoded JSON should contain 'auths' key") + } + if _, ok := auths["registry.example.com"]; !ok { + t.Errorf("Expected auths to be keyed by the registry host 'registry.example.com', got %v", auths) } } @@ -89,8 +99,126 @@ func TestGetAndVerifyCredentialsNoCredentials(t *testing.T) { da := New(log) da.skipCredVerification = true // Skip verification in tests - _, err := da.GetAndVerifyCredentials() + _, err := da.GetAndVerifyCredentials(context.Background(), constants.DefaultRegistry) if err == nil { t.Error("Expected error when no credentials are available") } } + +func TestRegistryRequiresAuth(t *testing.T) { + tests := []struct { + name string + challengeAuth bool // whether /v2/ demands a Bearer challenge at all + tokenStatus int // status the token endpoint returns, if challenged + tagsListStatus int // status the tags-list request returns + expectedRequires bool + }{ + { + name: "no auth mechanism: public", + challengeAuth: false, + tagsListStatus: http.StatusOK, + expectedRequires: false, + }, + { + name: "anonymous token granted, public repository", + challengeAuth: true, + tokenStatus: http.StatusOK, + tagsListStatus: http.StatusOK, + expectedRequires: false, + }, + { + name: "anonymous token granted, private repository", + challengeAuth: true, + tokenStatus: http.StatusOK, + tagsListStatus: http.StatusUnauthorized, + expectedRequires: true, + }, + { + name: "anonymous token granted, private repository hidden behind 404", + // Some registries return 404 instead of 401/403 for private + // repositories, to avoid leaking their existence to unauthenticated + // callers. + challengeAuth: true, + tokenStatus: http.StatusOK, + tagsListStatus: http.StatusNotFound, + expectedRequires: true, + }, + { + name: "anonymous token request itself rejected", + challengeAuth: true, + tokenStatus: http.StatusUnauthorized, + expectedRequires: true, + }, + { + name: "tags-list request fails with a server error", + // A transient 5xx doesn't tell us whether the registry is public or + // private, so this fails safe by reporting that auth is required, + // rather than treating it as confirmed "public". + challengeAuth: true, + tokenStatus: http.StatusOK, + tagsListStatus: http.StatusInternalServerError, + expectedRequires: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var registryAddr string + + mux := http.NewServeMux() + mux.HandleFunc("/v2/", func(w http.ResponseWriter, r *http.Request) { + if !tt.challengeAuth { + w.WriteHeader(http.StatusOK) + return + } + w.Header().Set("WWW-Authenticate", fmt.Sprintf(`Bearer realm="http://%s/token",service="test-registry"`, registryAddr)) + w.WriteHeader(http.StatusUnauthorized) + }) + mux.HandleFunc("/token", func(w http.ResponseWriter, r *http.Request) { + if tt.tokenStatus != http.StatusOK { + w.WriteHeader(tt.tokenStatus) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"token":"fake-anonymous-token"}`)) + }) + mux.HandleFunc("/v2/some-org/main/tags/list", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(tt.tagsListStatus) + }) + + server := httptest.NewServer(mux) + defer server.Close() + registryAddr = strings.TrimPrefix(server.URL, "http://") + + da := &DockerAuth{logger: logger.New()} + requiresAuth := da.RegistryRequiresAuth(context.Background(), registryAddr+"/some-org") + assert.Equal(t, tt.expectedRequires, requiresAuth) + }) + } +} + +func TestSplitRegistryHost(t *testing.T) { + tests := []struct { + name string + registry string + expectedHost string + expectedPath string + }{ + {"default registry", constants.DefaultRegistry, "quay.io", "rhacs-eng"}, + {"quay.io with org", "quay.io/stackrox-io", "quay.io", "stackrox-io"}, + {"registry with port and nested path", "registry.io:5000/org/suborg", "registry.io:5000", "org/suborg"}, + {"just hostname", "justahost", "justahost", ""}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + host, path := splitRegistryHost(tt.registry) + if host != tt.expectedHost { + t.Errorf("splitRegistryHost(%q): expected host %q, got %q", tt.registry, tt.expectedHost, host) + } + if path != tt.expectedPath { + t.Errorf("splitRegistryHost(%q): expected path %q, got %q", tt.registry, tt.expectedPath, path) + } + }) + } +} diff --git a/internal/types/cluster_type.go b/internal/types/cluster_type.go index fce6b8f..de8d71b 100644 --- a/internal/types/cluster_type.go +++ b/internal/types/cluster_type.go @@ -77,7 +77,9 @@ func (ct *ClusterType) UnmarshalYAML(unmarshal func(any) error) error { return fmt.Errorf("unknown cluster type identifier: %q", s) } -func (ct ClusterType) NeedsPullSecrets() bool { +// NeedsDefaultRegistryPullSecrets reports whether this cluster type lacks +// auto-configured credentials for the default image registry (quay.io/rhacs-eng). +func (ct ClusterType) NeedsDefaultRegistryPullSecrets() bool { return ct != ClusterTypeInfraOpenShift4 } diff --git a/tests/e2e/custom_registry_test.go b/tests/e2e/custom_registry_test.go new file mode 100644 index 0000000..52ff0c0 --- /dev/null +++ b/tests/e2e/custom_registry_test.go @@ -0,0 +1,59 @@ +//go:build e2e + +package e2e + +import ( + "os" + "os/exec" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestDeployWithStackroxIORegistry verifies that roxie can deploy Central using +// the public quay.io/stackrox-io registry instead of the default quay.io/rhacs-eng. +func TestDeployWithStackroxIORegistry(t *testing.T) { + dumpClusterStateOnFailure(t) + + const stackroxIORegistry = "quay.io/stackrox-io" + + envrcFile, err := os.CreateTemp(t.TempDir(), ".envrc.roxie-test-*") + require.NoError(t, err) + envrcPath := envrcFile.Name() + envrcFile.Close() + + t.Log("=== Deploying central with quay.io/stackrox-io registry ===") + args := append([]string{ + roxieBinary, "deploy", "--early-readiness", "central", + "--set", "roxie.imageRegistry=" + stackroxIORegistry, + "--envrc", envrcPath, + }, commonDeployArgs...) + runCommand(t, deployTimeout, nil, args...) + + verifyCentralInstalled(t, centralNamespace) + verifyOperatorDeploymentExists(t, operatorSystemNamespace) + verifyOperatorImageRegistry(t, operatorSystemNamespace, stackroxIORegistry) + + t.Log("=== Cleaning up ===") + teardownArgs := []string{roxieBinary, "teardown", "--skip-user-config", "central"} + runCommand(t, teardownTimeout, nil, teardownArgs...) + + verifyCentralNotInstalled(t, centralNamespace) +} + +// verifyOperatorImageRegistry asserts that the operator deployment's image is +// hosted on the expected registry. +func verifyOperatorImageRegistry(t *testing.T, namespace, expectedRegistry string) { + t.Helper() + + cmd := exec.Command("kubectl", "get", "deployment", operatorDeploymentName, "-n", namespace, + "-o", "jsonpath={.spec.template.spec.containers[0].image}") + output, err := cmd.Output() + require.NoErrorf(t, err, "Failed to get operator image in namespace %s", namespace) + + image := strings.TrimSpace(string(output)) + require.Truef(t, strings.HasPrefix(image, expectedRegistry+"/"), + "Expected operator image to be pulled from %s, got: %s", expectedRegistry, image) + t.Logf("✓ Operator image %s uses registry %s", image, expectedRegistry) +}