diff --git a/.mockery.yaml b/.mockery.yaml index 6e50ddb0c..c7d7fc37d 100644 --- a/.mockery.yaml +++ b/.mockery.yaml @@ -16,7 +16,6 @@ packages: interfaces: RegistryHandler: ImageHandler: - DAGCheck: DockerRegistryAPI: github.com/astronomer/astro-cli/internal/platform/astro/clients/astrov1: config: diff --git a/README.md b/README.md index 0b011069c..cfb6aa460 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ astro local check Validate this project's DAGs without starting Airflow (--s astro local upgrade airflow [version] Move the project to a new Airflow, the newest of its generation if none is given (--with-otto hands the rest to Otto) ``` -`astro start`, `astro stop`, and `astro logs` work as shorthand for the `local` versions. The cloud commands — `astro login`, `astro deploy`, `astro deployment`, `astro workspace` — are unchanged from v1. +`astro start`, `astro stop`, and `astro logs` work as shorthand for the `local` versions. The cloud commands — `astro login`, `astro deployment`, `astro workspace` — work as they did in v1. `astro deploy` ships a project with a `pyproject.toml`: a project made by Astro CLI 1.x is converted with `astro init` first, or keeps deploying with Astro CLI 1.x. Every command takes `--output json` for scripting; streaming commands like `logs` emit one JSON object per line. diff --git a/airflow/constants.go b/airflow/constants.go index 8fba31950..b1a096ea5 100644 --- a/airflow/constants.go +++ b/airflow/constants.go @@ -4,6 +4,3 @@ const ( QuayBaseImageName = "quay.io/astronomer" AstroImageRegistryBaseImageName = "astrocrpublic.azurecr.io" ) - -// DefaultTestPath is the DAG integrity test a deploy's parse step runs. -const DefaultTestPath = ".astro/test_dag_integrity_default.py" diff --git a/airflow/container.go b/airflow/container.go index aa80c6826..5b054b3a3 100644 --- a/airflow/container.go +++ b/airflow/container.go @@ -1,26 +1,11 @@ package airflow import ( - "crypto/md5" //nolint:gosec // reviewed; not a new risk in this shell code - "fmt" - "regexp" - "strings" - "github.com/docker/docker/client" - "github.com/pkg/errors" "github.com/astronomer/astro-cli/airflow/types" - "github.com/astronomer/astro-cli/config" - "github.com/astronomer/astro-cli/pkg/fileutil" ) -// DAGCheck checks a project's DAGs in its image, the parse and -// pytest steps of `astro deploy`. -type DAGCheck interface { - Pytest(pytestFile, customImageName, deployImageName, pytestArgsString string, buildSecrets []string) (string, error) - Parse(customImageName, deployImageName string, buildSecrets []string) error -} - // RegistryHandler defines methods require to handle all operations with registry type RegistryHandler interface { Login(username, token string) error @@ -32,17 +17,12 @@ type ImageHandler interface { Push(remoteImage, username, token string, getImageRepoSha bool) (string, error) GetLabel(altImageName, labelName string) (string, error) TagLocalImage(localImage string) error - Pytest(pytestFile, airflowHome, envFile, testHomeDirectory string, pytestArgs []string, htmlReport bool, config types.ImageBuildConfig) (string, error) } type DockerRegistryAPI interface { client.APIClient } -func DAGCheckInit(airflowHome, envFile, dockerfile, projectName string) (DAGCheck, error) { - return NewDAGChecker(airflowHome, envFile, dockerfile, projectName) -} - func RegistryHandlerInit(registry string) (RegistryHandler, error) { return DockerRegistryInit(registry) } @@ -50,56 +30,3 @@ func RegistryHandlerInit(registry string) (RegistryHandler, error) { func ImageHandlerInit(image string) ImageHandler { return DockerImageInit(image) } - -// ProjectNameUnique creates a reasonably unique project name based on the hashed -// path of the project. This prevents collisions of projects with identical dir names -// in different paths. ie (~/dev/project1 vs ~/prod/project1) -func ProjectNameUnique() (string, error) { - projectName := config.CFG.ProjectName.GetString() - - pwd, err := fileutil.GetWorkingDir() - if err != nil { - return "", errors.Wrap(err, "error retrieving working directory") - } - - // #nosec - b := md5.Sum([]byte(pwd)) - s := fmt.Sprintf("%x", b[:]) - - return sanitizeImageName(projectName + "_" + s[0:6]), nil -} - -// validImageName matches docker's grammar for a single image-name path -// component: alphanumeric runs joined by single "." / "_" / "-" separators (or -// a double "__"), starting and ending with an alphanumeric. -var validImageName = regexp.MustCompile(`^[a-z0-9]+(?:(?:[._]|__|-+)[a-z0-9]+)*$`) - -// sanitizeImageName turns an arbitrary project name into a name docker will -// accept as an image tag. Names docker already accepts are returned unchanged, -// so cached image tags keep their names. Anything else is lowercased, has every -// run of non-alphanumeric characters collapsed to a single "-", and its leading -// and trailing separators trimmed. The result is always a non-empty valid name. -func sanitizeImageName(s string) string { - s = strings.ToLower(s) - if validImageName.MatchString(s) { - return s - } - - var b strings.Builder - prevSep := false - for _, r := range s { - if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') { - b.WriteRune(r) - prevSep = false - } else if !prevSep { - b.WriteByte('-') - prevSep = true - } - } - - out := strings.Trim(b.String(), "-") - if out == "" { - out = "project" - } - return out -} diff --git a/airflow/docker.go b/airflow/docker.go index 370a94849..7e3be5092 100644 --- a/airflow/docker.go +++ b/airflow/docker.go @@ -1,123 +1,6 @@ package airflow -import ( - "fmt" - "strconv" - "strings" - - "github.com/pkg/errors" - - airflowTypes "github.com/astronomer/astro-cli/airflow/types" - "github.com/astronomer/astro-cli/pkg/ansi" - "github.com/astronomer/astro-cli/pkg/util" -) - const ( RuntimeImageLabel = "io.astronomer.docker.runtime.version" - pytestDirectory = "tests" componentName = "airflow" ) - -// DAGChecker runs a project's DAG checks in its image: the parse and -// pytest steps of a 1.x project's deploy. It is what is left of the 1.x -// `astro dev` container handler, which drove docker compose. -type DAGChecker struct { - airflowHome string - envFile string - dockerfile string - imageHandler ImageHandler -} - -func NewDAGChecker(airflowHome, envFile, dockerfile, imageName string) (*DAGChecker, error) { - if imageName == "" { - // Get project name from config - projectName, err := ProjectNameUnique() - if err != nil { - return nil, fmt.Errorf("error retrieving working directory: %w", err) - } - imageName = projectName - } - - return &DAGChecker{ - airflowHome: airflowHome, - envFile: envFile, - dockerfile: dockerfile, - imageHandler: DockerImageInit(ImageName(imageName, "latest")), - }, nil -} - -// Pytest creates and runs a container containing the users airflow image, requirments, packages, and volumes(DAGs folder, etc...) -// These containers runs pytest on a specified pytest file (pytestFile). A deploy's --pytest and --parse use it -func (d *DAGChecker) Pytest(pytestFile, customImageName, deployImageName, pytestArgsString string, buildSecrets []string) (string, error) { - // deployImageName may be provided to the function if it is being used in the deploy command - if deployImageName == "" { - // build image - if customImageName == "" { - err := d.imageHandler.Build(d.dockerfile, buildSecrets, airflowTypes.ImageBuildConfig{Path: d.airflowHome}) - if err != nil { - return "", err - } - } else { - // skip build if an customImageName is passed - err := d.imageHandler.TagLocalImage(customImageName) - if err != nil { - return "", err - } - } - } - - // determine pytest args and file - pytestArgs := strings.Fields(pytestArgsString) - - // Determine pytest file - if pytestFile != DefaultTestPath { - if !strings.Contains(pytestFile, pytestDirectory) { - pytestFile = pytestDirectory + "/" + pytestFile - } else if pytestFile == "" { - pytestFile = pytestDirectory + "/" - } - } - - // run pytests - exitCode, err := d.imageHandler.Pytest(pytestFile, d.airflowHome, d.envFile, "", pytestArgs, false, airflowTypes.ImageBuildConfig{Path: d.airflowHome}) - if err != nil { - return exitCode, err - } - if code, convErr := strconv.Atoi(exitCode); convErr == nil && code == 0 { // exit code 0 means the pytests passed - return "", nil - } - return exitCode, errors.New("something went wrong while Pytesting your Dags") -} - -func (d *DAGChecker) Parse(customImageName, deployImageName string, buildSecrets []string) error { - // check for file - path := d.airflowHome + "/" + DefaultTestPath - - fileExist, err := util.Exists(path) - if err != nil { - return err - } - if !fileExist { - // Only a 1.x project deploys with --parse, and Astro CLI 1.x's - // `astro dev init` wrote this file into it; v2 writes it nowhere. A - // project without it has never had a parse check to run, so the deploy - // goes on, as it always has, but says so. - fmt.Println("\nSkipping the DAG parse check: it runs " + path + ", which this project does not have. " + - "Astro CLI 1.x's `astro dev init` created that file; add it back to the project to run the check.") - - return nil - } - - fmt.Println("Checking your Dags for errors…") - - pytestFile := DefaultTestPath - exitCode, err := d.Pytest(pytestFile, customImageName, deployImageName, "", buildSecrets) - if err != nil { - if code, convErr := strconv.Atoi(exitCode); convErr == nil && code == 1 { // exit code 1 means tests failed - return errors.New("See above for errors detected in your Dags") - } - return errors.Wrap(err, "something went wrong while parsing your Dags") - } - fmt.Println(ansi.Green("✔") + " No errors detected in your Dags ") - return err -} diff --git a/airflow/docker_image.go b/airflow/docker_image.go index 6bbbce0a0..b8d336cf1 100644 --- a/airflow/docker_image.go +++ b/airflow/docker_image.go @@ -27,7 +27,6 @@ import ( "github.com/astronomer/astro-cli/config" "github.com/astronomer/astro-cli/pkg/logger" "github.com/astronomer/astro-cli/pkg/spinner" - "github.com/astronomer/astro-cli/pkg/util" ) const ( @@ -182,103 +181,6 @@ func (d *DockerImage) Build(dockerfilePath string, buildSecrets []string, buildC return nil } -func (d *DockerImage) Pytest(pytestFile, airflowHome, envFile, testHomeDirectory string, pytestArgs []string, htmlReport bool, buildConfig airflowTypes.ImageBuildConfig) (string, error) { - // delete container - containerRuntime, err := runtimes.GetContainerRuntimeBinary() - if err != nil { - return "", err - } - err = cmdExec(containerRuntime, nil, nil, "rm", "astro-pytest") - if err != nil { - logger.Debug(err) - } - // Change to location of Dockerfile - err = os.Chdir(buildConfig.Path) - if err != nil { - return "", err - } - args := []string{ - "create", - "-i", - "--name", - "astro-pytest", - } - fileExist, err := util.Exists(airflowHome + "/" + envFile) - if err != nil { - return "", err - } - if fileExist { - args = append(args, []string{"--env-file", envFile}...) - } - args = append(args, []string{d.imageName, "pytest", pytestFile}...) - args = append(args, pytestArgs...) - // run pytest image - var stdout, stderr io.Writer - if logger.IsLevelEnabled(logrus.WarnLevel) { - stdout = os.Stdout - stderr = os.Stderr - } else { - stdout = nil - stderr = nil - } - - // create pytest container - err = cmdExec(containerRuntime, stdout, stderr, args...) - if err != nil { - return "", err - } - - // Copy host directories into the container using docker cp. - // This ensures fresh files from the host are used (not stale from - // image build cache) and works with remote Docker daemons (CI). - copyDirs := []string{"dags", "tests", "plugins", "include", ".astro"} - for _, dir := range copyDirs { - srcPath := airflowHome + "/" + dir - if exists, _ := util.Exists(srcPath); !exists { //nolint:errcheck // treated as absent on error - continue - } - docErr := cmdExec(containerRuntime, stdout, stderr, "cp", srcPath, "astro-pytest:/usr/local/airflow/") - if docErr != nil { - return "", docErr - } - } - - // start pytest container - err = cmdExec(containerRuntime, stdout, stderr, []string{"start", "astro-pytest", "-a"}...) - if err != nil { - logger.Debugf("Error starting pytest container: %s", err.Error()) - } - - // get exit code - args = []string{ - "inspect", - "astro-pytest", - "--format={{.State.ExitCode}}", - } - var outb bytes.Buffer - inspectErr := cmdExec(containerRuntime, &outb, stderr, args...) - if inspectErr != nil { - logger.Debug(inspectErr) - } - - if htmlReport { - // Copy the dag-test-report.html file from the container to the destination folder - cpErr := cmdExec(containerRuntime, nil, stderr, "cp", "astro-pytest:/usr/local/airflow/dag-test-report.html", "./"+testHomeDirectory) - if cpErr != nil { - logger.Debugf("Error copying dag-test-report.html file from the pytest container: %s", cpErr.Error()) - } - } - - // delete container - rmErr := cmdExec(containerRuntime, nil, stderr, "rm", "astro-pytest") - if rmErr != nil { - logger.Debugf("Error removing the astro-pytest container: %s", rmErr.Error()) - } - - // trim the trailing newline so consumers get a clean integer string to parse - return strings.TrimSpace(outb.String()), err -} - func (d *DockerImage) Push(remoteImage, username, token string, getImageRepoSha bool) (string, error) { containerRuntime, err := runtimes.GetContainerRuntimeBinary() if err != nil { diff --git a/airflow/docker_image_test.go b/airflow/docker_image_test.go index 035663d0f..ce7dca977 100644 --- a/airflow/docker_image_test.go +++ b/airflow/docker_image_test.go @@ -38,7 +38,7 @@ func countOccurrences(haystack []string, needle string) (count int) { // buildContext is a project directory of the test's own, holding the // Dockerfile and .dockerignore a build reads, so nothing depends on what -// other tests left in the package directory. Build and Pytest chdir into the +// other tests left in the package directory. Build chdirs into the // build context, so the test starts there too and t.Chdir puts the package // directory back when it ends. func buildContext(s *Suite) string { @@ -176,118 +176,6 @@ FROM quay.io/astronomer/astro-runtime:12.0.0` }) } -func (s *Suite) TestDockerImagePytest() { - handler := DockerImage{ - imageName: "testing", - } - - cwd := buildContext(s) - var err error - - options := airflowTypes.ImageBuildConfig{ - Path: cwd, - TargetPlatforms: []string{"linux/amd64"}, - NoCache: false, - } - - s.Run("pytest success", func() { - cmdExec = func(cmd string, stdout, stderr io.Writer, args ...string) error { - return nil - } - _, err = handler.Pytest("", "", "", "", []string{}, true, options) - s.NoError(err) - }) - - s.Run("create error", func() { - cmdExec = func(cmd string, stdout, stderr io.Writer, args ...string) error { - switch { - case args[0] == "create": - return errMock - default: - return nil - } - } - _, err = handler.Pytest("", "", "", "", []string{}, true, options) - s.Error(err) - }) - - s.Run("start error", func() { - cmdExec = func(cmd string, stdout, stderr io.Writer, args ...string) error { - switch { - case args[0] == "start": - return errMock - case args[0] == "inspect": - stdout.Write([]byte(`exit code 1`)) // making sure exit code is captured properly - return nil - default: - return nil - } - } - out, err := handler.Pytest("", "", "", "", []string{}, true, options) - s.Error(err) - s.Equal(out, "exit code 1") - }) - - s.Run("exit code trimmed to clean integer", func() { - cmdExec = func(cmd string, stdout, stderr io.Writer, args ...string) error { - switch { - case args[0] == "start": - return errMock - case args[0] == "inspect": - stdout.Write([]byte("10\n")) // docker inspect prints the code with a trailing newline - return nil - default: - return nil - } - } - out, err := handler.Pytest("", "", "", "", []string{}, true, options) - s.Error(err) - s.Equal("10", out) - }) - - s.Run("copy error", func() { - // Create a directory so the docker cp loop has something to copy - dagsDir := cwd + "/dags" - s.NoError(os.MkdirAll(dagsDir, os.ModePerm)) - defer os.RemoveAll(dagsDir) - - cmdExec = func(cmd string, stdout, stderr io.Writer, args ...string) error { - switch { - case args[0] == "cp": - return errMock - default: - return nil - } - } - _, err = handler.Pytest("", cwd, "", "", []string{}, true, options) - s.Error(err) - }) - - s.Run("pytest error", func() { - options = airflowTypes.ImageBuildConfig{ - Path: cwd, - TargetPlatforms: []string{"linux/amd64"}, - NoCache: false, - } - - cmdExec = func(cmd string, stdout, stderr io.Writer, args ...string) error { - return errMock - } - _, err = handler.Pytest("", "", "", "", []string{}, false, options) - s.Contains(err.Error(), errMock.Error()) - }) - s.Run("unable to read file error", func() { - options := airflowTypes.ImageBuildConfig{ - Path: "incorrect-path", - TargetPlatforms: []string{"linux/amd64"}, - NoCache: false, - } - - _, err = handler.Pytest("", "", "", "", []string{}, false, options) - s.Error(err) - }) -} - func (s *Suite) TestDockerImagePush() { handler := DockerImage{ imageName: "testing", diff --git a/airflow/docker_test.go b/airflow/docker_test.go index e368ffdb2..574154880 100644 --- a/airflow/docker_test.go +++ b/airflow/docker_test.go @@ -1,15 +1,7 @@ package airflow import ( - "io" - "os" - "path/filepath" - "github.com/pkg/errors" - "github.com/stretchr/testify/mock" - - "github.com/astronomer/astro-cli/airflow/mocks" - airflowTypes "github.com/astronomer/astro-cli/airflow/types" ) var errMockDocker = errors.New("mock docker compose error") @@ -21,200 +13,3 @@ func (s *Suite) TestRepositoryName() { func (s *Suite) TestImageName() { s.Equal(ImageName("test-repo", "0.15.0"), "test-repo/airflow:0.15.0") } - -func (s *Suite) TestSanitizeImageName() { - // Already-valid names must pass through byte-for-byte. Changing them would - // rename cached image tags and orphan users' local images. - unchanged := []string{ - "simple", - "project_name", - "my-project", - "my-project_abc123", // typical ProjectNameUnique output - "a__b", // docker allows a double-underscore separator - "foo.bar", - "tmp155bkx9_684ec5", - } - for _, in := range unchanged { - s.Equal(in, sanitizeImageName(in), "valid name %q must not change", in) - } - - // Malformed names must become valid docker image names. These are the - // shapes that broke `astro dev` builds from random temp-dir names. - fixes := map[string]string{ - "a-_b": "a-b", // the reported "-_" double separator - "__x": "x", // leading separators trimmed - "-lead": "lead", // leading separator trimmed - "trail-": "trail", // trailing separator trimmed - "!!!": "project", // all punctuation -> non-empty fallback - "tmp-155-bkx-9-_684ec5": "tmp-155-bkx-9-684ec5", // the exact failing shape - "UPPER": "upper", // docker image names are lowercase - } - for in, want := range fixes { - got := sanitizeImageName(in) - s.Equal(want, got, "input %q", in) - s.True(validImageName.MatchString(got), "sanitized %q -> %q must be a valid image name", in, got) - } - - // Whatever the input, the result is always a valid, non-empty image name. - for _, in := range []string{"", "-", "___", ".-.", "a-_-_-b", "9", "-_-"} { - got := sanitizeImageName(in) - s.NotEmpty(got) - s.True(validImageName.MatchString(got), "sanitized %q -> %q must be a valid image name", in, got) - } -} - -func (s *Suite) TestNewDAGChecker() { - _, err := NewDAGChecker(s.T().TempDir(), "", "Dockerfile", "") - s.NoError(err) -} - -var errExecMock = errors.New("docker is not running") - -func (s *Suite) TestDAGCheckerPytest() { - checker := DAGChecker{} - s.Run("success", func() { - imageHandler := new(mocks.ImageHandler) - imageHandler.On("Build", "", mock.Anything, airflowTypes.ImageBuildConfig{Path: checker.airflowHome, NoCache: false}).Return(nil).Once() - imageHandler.On("Pytest", mock.Anything, mock.Anything, mock.Anything, mock.Anything, []string{}, mock.Anything, airflowTypes.ImageBuildConfig{Path: checker.airflowHome, NoCache: false}).Return("0", nil).Once() - - checker.imageHandler = imageHandler - - resp, err := checker.Pytest("", "", "", "", nil) - - s.NoError(err) - s.Equal("", resp) - imageHandler.AssertExpectations(s.T()) - }) - - s.Run("success custom image", func() { - imageHandler := new(mocks.ImageHandler) - imageHandler.On("TagLocalImage", mock.Anything).Return(nil) - imageHandler.On("Pytest", mock.Anything, mock.Anything, mock.Anything, mock.Anything, []string{}, mock.Anything, airflowTypes.ImageBuildConfig{Path: checker.airflowHome, NoCache: false}).Return("0", nil).Once() - - checker.imageHandler = imageHandler - - resp, err := checker.Pytest("", "custom-image-name", "", "", nil) - - s.NoError(err) - s.Equal("", resp) - imageHandler.AssertExpectations(s.T()) - }) - - s.Run("unexpected exit code", func() { - imageHandler := new(mocks.ImageHandler) - imageHandler.On("Build", "", mock.Anything, airflowTypes.ImageBuildConfig{Path: checker.airflowHome, NoCache: false}).Return(nil).Once() - imageHandler.On("Pytest", mock.Anything, mock.Anything, mock.Anything, mock.Anything, []string{}, mock.Anything, airflowTypes.ImageBuildConfig{Path: checker.airflowHome, NoCache: false}).Return("1", nil).Once() - - mockResponse := "1" - checker.imageHandler = imageHandler - - resp, err := checker.Pytest("", "", "", "", nil) - s.Contains(err.Error(), "something went wrong while Pytesting your Dags") - s.Equal(mockResponse, resp) - imageHandler.AssertExpectations(s.T()) - }) - - s.Run("internal error exit code 10 reported as failure", func() { - // exit code 10 substring-contains "0"; the old check reported it as a pass - imageHandler := new(mocks.ImageHandler) - imageHandler.On("Build", "", mock.Anything, airflowTypes.ImageBuildConfig{Path: checker.airflowHome, NoCache: false}).Return(nil).Once() - imageHandler.On("Pytest", mock.Anything, mock.Anything, mock.Anything, mock.Anything, []string{}, mock.Anything, airflowTypes.ImageBuildConfig{Path: checker.airflowHome, NoCache: false}).Return("10", nil).Once() - - checker.imageHandler = imageHandler - - resp, err := checker.Pytest("", "", "", "", nil) - s.Contains(err.Error(), "something went wrong while Pytesting your Dags") - s.Equal("10", resp) - imageHandler.AssertExpectations(s.T()) - }) - - s.Run("interrupt exit code 130 reported as failure", func() { - imageHandler := new(mocks.ImageHandler) - imageHandler.On("Build", "", mock.Anything, airflowTypes.ImageBuildConfig{Path: checker.airflowHome, NoCache: false}).Return(nil).Once() - imageHandler.On("Pytest", mock.Anything, mock.Anything, mock.Anything, mock.Anything, []string{}, mock.Anything, airflowTypes.ImageBuildConfig{Path: checker.airflowHome, NoCache: false}).Return("130", nil).Once() - - checker.imageHandler = imageHandler - - resp, err := checker.Pytest("", "", "", "", nil) - s.Contains(err.Error(), "something went wrong while Pytesting your Dags") - s.Equal("130", resp) - imageHandler.AssertExpectations(s.T()) - }) - - s.Run("image build failure", func() { - imageHandler := new(mocks.ImageHandler) - imageHandler.On("Build", "", mock.Anything, airflowTypes.ImageBuildConfig{Path: checker.airflowHome, NoCache: false}).Return(errMockDocker).Once() - - checker.imageHandler = imageHandler - - _, err := checker.Pytest("", "", "", "", nil) - s.ErrorIs(err, errMockDocker) - imageHandler.AssertExpectations(s.T()) - }) -} - -// parseProject is a project directory holding the DAG integrity test that -// Parse runs, at the path a 1.x project keeps it. -func parseProject(s *Suite) string { - dir := s.T().TempDir() - s.Require().NoError(os.MkdirAll(filepath.Join(dir, ".astro"), 0o755)) - s.Require().NoError(os.WriteFile(filepath.Join(dir, DefaultTestPath), []byte("def test_dags(): pass\n"), 0o600)) - return dir -} - -func (s *Suite) TestDAGCheckerParse() { - for _, tc := range []struct { - name string - exitCode string - wantErr string - }{ - {"success", "0", ""}, - {"exit code 1", "1", "See above for errors detected in your Dags"}, - {"exit code 2", "2", "something went wrong while parsing your Dags"}, - // exit codes 10 and 130 (Ctrl-C) substring-contain "1"; the old check - // reported them as a clean DAG error - {"internal error exit code 10", "10", "something went wrong while parsing your Dags"}, - {"interrupt exit code 130", "130", "something went wrong while parsing your Dags"}, - } { - s.Run(tc.name, func() { - project := parseProject(s) - imageHandler := new(mocks.ImageHandler) - // The integrity test is run where it is, not looked for under tests/. - imageHandler.On("Pytest", DefaultTestPath, project, mock.Anything, mock.Anything, []string{}, mock.Anything, airflowTypes.ImageBuildConfig{Path: project, NoCache: false}).Return(tc.exitCode, nil).Once() - checker := DAGChecker{airflowHome: project, imageHandler: imageHandler} - - err := checker.Parse("", "test", nil) - if tc.wantErr == "" { - s.NoError(err) - } else { - s.ErrorContains(err, tc.wantErr) - } - imageHandler.AssertExpectations(s.T()) - }) - } - - s.Run("file does not exists", func() { - checker := DAGChecker{airflowHome: s.T().TempDir(), imageHandler: new(mocks.ImageHandler)} - - r, w, _ := os.Pipe() - os.Stdout = w - - err := checker.Parse("", "test", nil) - s.NoError(err) - - w.Close() - out, _ := io.ReadAll(r) - - s.Contains(string(out), "Skipping the DAG parse check") - s.Contains(string(out), "add it back to the project") - s.NotContains(string(out), "astro dev parse", "astro dev parse does not exist in v2") - s.NotContains(string(out), "run `astro dev init`", "astro dev init does not exist in v2") - }) - - s.Run("invalid file name", func() { - checker := DAGChecker{airflowHome: "\x0004"} - - err := checker.Parse("", "test", nil) - s.ErrorContains(err, "invalid argument") - }) -} diff --git a/airflow/mocks/DAGCheck.go b/airflow/mocks/DAGCheck.go deleted file mode 100644 index 5d7c9bcf2..000000000 --- a/airflow/mocks/DAGCheck.go +++ /dev/null @@ -1,70 +0,0 @@ -// Code generated by mockery v2.50.0. DO NOT EDIT. - -package mocks - -import mock "github.com/stretchr/testify/mock" - -// DAGCheck is an autogenerated mock type for the DAGCheck type -type DAGCheck struct { - mock.Mock -} - -// Parse provides a mock function with given fields: customImageName, deployImageName, buildSecrets -func (_m *DAGCheck) Parse(customImageName string, deployImageName string, buildSecrets []string) error { - ret := _m.Called(customImageName, deployImageName, buildSecrets) - - if len(ret) == 0 { - panic("no return value specified for Parse") - } - - var r0 error - if rf, ok := ret.Get(0).(func(string, string, []string) error); ok { - r0 = rf(customImageName, deployImageName, buildSecrets) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// Pytest provides a mock function with given fields: pytestFile, customImageName, deployImageName, pytestArgsString, buildSecrets -func (_m *DAGCheck) Pytest(pytestFile string, customImageName string, deployImageName string, pytestArgsString string, buildSecrets []string) (string, error) { - ret := _m.Called(pytestFile, customImageName, deployImageName, pytestArgsString, buildSecrets) - - if len(ret) == 0 { - panic("no return value specified for Pytest") - } - - var r0 string - var r1 error - if rf, ok := ret.Get(0).(func(string, string, string, string, []string) (string, error)); ok { - return rf(pytestFile, customImageName, deployImageName, pytestArgsString, buildSecrets) - } - if rf, ok := ret.Get(0).(func(string, string, string, string, []string) string); ok { - r0 = rf(pytestFile, customImageName, deployImageName, pytestArgsString, buildSecrets) - } else { - r0 = ret.Get(0).(string) - } - - if rf, ok := ret.Get(1).(func(string, string, string, string, []string) error); ok { - r1 = rf(pytestFile, customImageName, deployImageName, pytestArgsString, buildSecrets) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// NewDAGCheck creates a new instance of DAGCheck. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewDAGCheck(t interface { - mock.TestingT - Cleanup(func()) -}) *DAGCheck { - mock := &DAGCheck{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/airflow/mocks/ImageHandler.go b/airflow/mocks/ImageHandler.go index 89c90b0e1..1949bf48d 100644 --- a/airflow/mocks/ImageHandler.go +++ b/airflow/mocks/ImageHandler.go @@ -86,34 +86,6 @@ func (_m *ImageHandler) Push(remoteImage string, username string, token string, return r0, r1 } -// Pytest provides a mock function with given fields: pytestFile, airflowHome, envFile, testHomeDirectory, pytestArgs, htmlReport, config -func (_m *ImageHandler) Pytest(pytestFile string, airflowHome string, envFile string, testHomeDirectory string, pytestArgs []string, htmlReport bool, config types.ImageBuildConfig) (string, error) { - ret := _m.Called(pytestFile, airflowHome, envFile, testHomeDirectory, pytestArgs, htmlReport, config) - - if len(ret) == 0 { - panic("no return value specified for Pytest") - } - - var r0 string - var r1 error - if rf, ok := ret.Get(0).(func(string, string, string, string, []string, bool, types.ImageBuildConfig) (string, error)); ok { - return rf(pytestFile, airflowHome, envFile, testHomeDirectory, pytestArgs, htmlReport, config) - } - if rf, ok := ret.Get(0).(func(string, string, string, string, []string, bool, types.ImageBuildConfig) string); ok { - r0 = rf(pytestFile, airflowHome, envFile, testHomeDirectory, pytestArgs, htmlReport, config) - } else { - r0 = ret.Get(0).(string) - } - - if rf, ok := ret.Get(1).(func(string, string, string, string, []string, bool, types.ImageBuildConfig) error); ok { - r1 = rf(pytestFile, airflowHome, envFile, testHomeDirectory, pytestArgs, htmlReport, config) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - // TagLocalImage provides a mock function with given fields: localImage func (_m *ImageHandler) TagLocalImage(localImage string) error { ret := _m.Called(localImage) diff --git a/cmd/apc/deploy.go b/cmd/apc/deploy.go index 81731d527..1d30176ac 100644 --- a/cmd/apc/deploy.go +++ b/cmd/apc/deploy.go @@ -17,19 +17,15 @@ import ( ) var ( - forceDeploy bool - forcePrompt bool - saveDeployConfig bool - deployOutput cliout.Format - deployYes bool - - ignoreCacheDeploy = false + forceDeploy bool + forcePrompt bool + deployOutput cliout.Format + deployYes bool // hasUncommittedChanges is a variable so a test does not depend on the // state of the checkout it runs in. hasUncommittedChanges = git.HasUncommittedChanges - EnsureProjectDir = utils.EnsureDockerfileProjectDir DeployAirflowImage = deploy.Airflow DagsOnlyDeploy = deploy.DagsOnlyDeploy UpdateDeploymentImage = deploy.UpdateDeploymentImage @@ -43,13 +39,10 @@ var ( ErrImageNameNotPassedForRemoteFlag = errors.New("--image-name is mandatory when --remote flag is passed") ) -var deployExample = ` # Deploy this project, picking the Deployment from a list - astro deploy - - # Deploy to a given Deployment - astro deploy +var deployExample = ` # Deploy an image built on this machine, picking the Deployment from a list + astro deploy --image-name - # Deploy a custom image built on this machine + # Deploy an image built on this machine to a given Deployment astro deploy --image-name ` // The warnings a deploy can give about its DAGs. @@ -60,36 +53,44 @@ const ( // The image may well carry DAGs (one built from a Dockerfile that copies // them does); one astro package generated the build for does not. warningImageNameDagsInImage = "this Deployment runs the Dags inside the image %s; the dags folder is not uploaded. An image astro package built without a dockerfile declared under [tool.astro] contains none." - // noticeNoDagsDir: there is no dags directory to upload, so + // noticeNoDagsDir: the project has no dags directory to upload, so // DagsOnlyDeploy uploaded nothing. An empty upload would have deleted - // the Deployment's DAGs. The %s after the path is the advice: where to - // run the deploy from, or what the project lacks. - noticeNoDagsDir = "no Dags were uploaded: there is no dags directory in %s, and the Deployment keeps the Dags it had. %s" - // adviceRunFromProject: --image-name skips the project check, so the - // deploy may not have been run from one. - adviceRunFromProject = "To upload them, run the deploy from the project directory." - // adviceCreateDagsDir: the deploy ran from a project, which has none. - adviceCreateDagsDir = "To upload Dags, create a dags directory in the project." + // the Deployment's DAGs. + noticeNoDagsDir = "no Dags were uploaded: there is no dags directory in %s, and the Deployment keeps the Dags it had. To upload Dags, create a dags directory in the project." + // noticeImageAlone: an --image-name deploy outside a pyproject.toml + // project ships the image alone, to a Deployment that takes DAG + // uploads, so its DAGs are as they were. + noticeImageAlone = "Dags were not updated: this deploy ran outside a pyproject.toml project, so it shipped the image alone, and the Deployment keeps the Dags it had. To upload them, run astro deploy %s --dags from the project directory." + // noticeImageAloneUnplaced: the same, to a Deployment whose DAG source + // could not be read. + noticeImageAloneUnplaced = "this deploy ran outside a pyproject.toml project, so it shipped the image alone. If this Deployment takes Dag uploads, it keeps the Dags it had: to upload them, run astro deploy %s --dags from the project directory." // noticeDagsUndecided: the image was deployed, and the cluster config // that says whether the Deployment takes DAG uploads could not be read, // so an upload that may have been due did not happen. noticeDagsUndecided = "Dags were NOT updated: whether this Deployment takes Dag uploads could not be read (%v). The image was deployed, and the Deployment keeps the Dags it had. To upload them, run astro deploy %s --dags from the project directory." ) -var errUncommittedChanges = errors.New("project directory has uncommitted changes: commit them, or use `astro deploy --force` to deploy anyway") +// What a deploy that would build the project is told: v2 builds no project +// for APC. Astro CLI 1.x keeps deploying projects there, and a pyproject.toml +// project gets an APC deploy of its own later. +const ( + errBuildDeployNoProject = "Astro CLI v2 cannot build and deploy projects to Astro Private Cloud yet: use Astro CLI 1.x for now. " + + "To deploy an image you built yourself, pass --image-name" + errBuildDeployManifest = "Astro CLI v2 cannot build and deploy projects to Astro Private Cloud yet: use Astro CLI 1.x for now. " + + "Support for pyproject.toml projects on Astro Private Cloud is coming. Until then, --dags uploads this project's DAGs, " + + "and a project that declares dockerfile under [tool.astro] (a Dockerfile FROM an Astro Runtime image) " + + "builds an image with astro package --tag that astro deploy --image-name deploys" +) + +var errUncommittedChanges = errors.New("project directory has uncommitted changes: commit them, or use astro deploy --force to deploy anyway") func NewDeployCmd(out io.Writer) *cobra.Command { cmd := &cobra.Command{ Use: "deploy [DEPLOYMENT_ID]", - Short: "Deploy an Airflow project", - Long: "Deploy an Airflow project to an APC Deployment", - Args: cobra.MaximumNArgs(1), - PreRunE: func(cmd *cobra.Command, args []string) error { - if cmd.Flags().Changed("image-name") { - return nil - } - return EnsureProjectDir(cmd, args) - }, + Short: "Deploy an image or Dags to a Deployment", + Long: "Deploy an image you built, or a project's Dags, to an APC Deployment. Astro CLI v2 does not build projects for Astro Private Cloud yet: " + + "to build and deploy a project there, use Astro CLI 1.x. A project in the Astro CLI 1.x layout (a Dockerfile and .astro/config.yaml) deploys with Astro CLI 1.x only.", + Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { return deployAirflow(cmd, args, out) }, @@ -97,23 +98,71 @@ func NewDeployCmd(out io.Writer) *cobra.Command { } cmd.Flags().BoolVarP(&forceDeploy, "force", "f", false, "Force deploy if uncommitted changes") cmd.Flags().BoolVarP(&forcePrompt, "prompt", "p", false, "Force prompt to choose target deployment") - cmd.Flags().BoolVarP(&saveDeployConfig, "save", "s", false, "Save deployment in config for future deploys") - cmd.Flags().BoolVarP(&ignoreCacheDeploy, "no-cache", "", false, "Do not use cache when building container image") cmd.Flags().StringVar(&workspaceID, "workspace-id", "", "Workspace assigned to the Deployment") cmd.Flags().StringVar(&description, "description", "", "Description to attach to the deploy, for traceability (default: one based on the deploy type)") cmd.Flags().BoolVarP(&isImageOnlyDeploy, "image", "", false, "Push only an image to your Deployment; works only for Dag-only, Git-sync-based and NFS-based Deployments") - cmd.Flags().StringVarP(&imageName, "image-name", "i", "", "Name of the custom image(should be present locally unless --remote is specified) to deploy") + cmd.Flags().StringVarP(&imageName, "image-name", "i", "", "Name of the custom image (present locally unless --remote is specified) to deploy") cmd.Flags().StringVar(&runtimeVersionForImageName, "runtime-version", "", "Runtime version of the image to deploy. Example - 12.1.1. Mandatory if --image-name --remote is provided") cmd.Flags().BoolVarP(&imagePresentOnRemote, "remote", "", false, "Custom image which is present on the remote registry. Can only be used with --image-name flag") cmd.Flags().BoolVarP(&deployYes, "yes", "y", false, "Answer the deploy's confirmations yes: an image tag that is not recommended, and a DAGs folder with no DAGs") cliout.AddOutputFlag(cmd, &deployOutput) if !context.IsCloudContext() && houston.VerifyVersionMatch(houstonVersion, houston.VersionRestrictions{GTE: "0.34.0"}) { - cmd.Flags().BoolVarP(&isDagOnlyDeploy, "dags", "d", false, "Push only Dags to your Deployment") + cmd.Flags().BoolVarP(&isDagOnlyDeploy, "dags", "d", false, "Push only the Dags of the pyproject.toml project here to your Deployment") } return cmd } +// refuseUndeployable stops a deploy v2 does not make on APC, before anything +// is asked or sent: +// +// - any deploy in or below a project in the Astro CLI 1.x layout, which +// Astro CLI 1.x deploys, or below a pyproject.toml project's root +// (kind no_project), --image-name included (utils.Locate's walk); +// - --dags anywhere but in a pyproject.toml project: it uploads that +// project's dags directory (kind no_project); +// - a deploy with no --image-name, which would build the project: v2 builds +// none for APC yet. From a pyproject.toml project that is a mode v2 does +// not have, a usage error; anywhere else there is no project to build, +// kind no_project, as astro deploy on Astro says. +// +// --image-name (with or without --remote) runs at a pyproject.toml project's +// root, where the project's DAGs are uploaded after it to a Deployment that +// takes them, and outside any project, where the image ships alone. +func refuseUndeployable() (inProject bool, err error) { + // One walk routes the deploy and words its refusal. + where := utils.Locate(config.WorkingPath) + inProject = where.AtRoot(config.WorkingPath) + if !inProject && !where.None() { + // In or below a 1.x project, or below a pyproject.toml project's + // root: every mode is refused, --image-name included, as on Astro. + return false, utils.NoDeployableProject(where, utils.Deploy1xRefusedAPC) + } + switch { + case isDagOnlyDeploy && imageName != "": + // As astro deploy says it: an image source makes no sense with a + // deploy that ships only DAGs. + return inProject, cliout.Usage(errDagsWithImageName) + case isDagOnlyDeploy: + if !inProject { + return false, utils.NoDeployableProject(where, utils.Deploy1xRefusedAPC) + } + case imagePresentOnRemote && imageName == "": + return inProject, ErrImageNameNotPassedForRemoteFlag + case imageName == "": + // The value, not whether the flag was given: an empty --image-name= + // names no image, so this would be a build. + if inProject { + return true, cliout.Usage(errors.New(errBuildDeployManifest)) + } + return false, utils.NoProject(errBuildDeployNoProject) + } + return inProject, nil +} + +// errDagsWithImageName is --dags with --image-name, in astro deploy's words. +var errDagsWithImageName = errors.New("--dags deploys only your DAGs; drop --image-name") + // The kinds of deploy deployJSON.Type names. const ( deployTypeImageAndDags = "image_and_dags" @@ -147,6 +196,14 @@ type deployJSON struct { } func deployAirflow(cmd *cobra.Command, args []string, out io.Writer) error { + inProject, err := refuseUndeployable() + if err != nil { + // Not a usage mistake to print the usage block under: what is in + // the directory decided it. + cmd.SilenceUsage = true + return err + } + ws, err := coalesceWorkspace() if err != nil { return fmt.Errorf("failed to find a valid workspace: %w", err) @@ -159,17 +216,11 @@ func deployAirflow(cmd *cobra.Command, args []string, out io.Writer) error { deploymentID = args[0] } - // Save release name in config if specified - if deploymentID != "" && saveDeployConfig { - err = config.CFG.ProjectDeployment.SetProjectString(deploymentID) - if err != nil { - return err - } - } - // An error, not a printed note: returning nil here made a deploy that never - // happened exit 0, so CI reported it as a success. - if hasUncommittedChanges("") && !forceDeploy { + // happened exit 0, so CI reported it as a success. Only a deploy that + // reads the project is held to it: an image deployed alone from outside + // one reads nothing from the directory. + if inProject && hasUncommittedChanges("") && !forceDeploy { // Not a usage mistake, so no usage block under the error. cmd.SilenceUsage = true return errUncommittedChanges @@ -206,9 +257,6 @@ func deployAirflow(cmd *cobra.Command, args []string, out io.Writer) error { } if imagePresentOnRemote { - if imageName == "" { - return ErrImageNameNotPassedForRemoteFlag - } deployed, err := UpdateDeploymentImage(houstonClient, deploymentID, ws, runtimeVersionForImageName, imageName, opts) if err != nil { return err @@ -217,7 +265,7 @@ func deployAirflow(cmd *cobra.Command, args []string, out io.Writer) error { result.Image, result.RuntimeVersion = imageName, runtimeVersionForImageName } else { // Since we prompt the user to enter the deploymentID in come cases for DeployAirflowImage, reusing the same deploymentID for DagsOnlyDeploy - deployed, err := DeployAirflowImage(houstonClient, config.WorkingPath, deploymentID, ws, ignoreCacheDeploy, forcePrompt, description, isImageOnlyDeploy, imageName, opts) + deployed, err := DeployAirflowImage(houstonClient, deploymentID, ws, forcePrompt, isImageOnlyDeploy, imageName, opts) if err != nil { return err } @@ -232,7 +280,7 @@ func deployAirflow(cmd *cobra.Command, args []string, out io.Writer) error { return emitDeploy(r, &result) } - after := &dagsAfterImage{client: houstonClient, workspace: ws, deployment: deploymentID, path: config.WorkingPath, imageName: imageName, description: description, dags: dags} + after := &dagsAfterImage{client: houstonClient, workspace: ws, deployment: deploymentID, path: config.WorkingPath, imageName: imageName, description: description, dags: dags, inProject: inProject} if err := deployDagsAfterImage(after, opts, &result); err != nil { return err } @@ -253,6 +301,9 @@ type dagsAfterImage struct { // dags is where the Deployment takes its DAGs from, as the image deploy // found it. dags deploy.DagsFrom + // inProject is whether the deploy ran at a pyproject.toml project's + // root, whose dags directory follows the image. + inProject bool } // deployDagsAfterImage uploads the working directory's DAGs to a Deployment @@ -261,18 +312,29 @@ type dagsAfterImage struct { func deployDagsAfterImage(a *dagsAfterImage, opts deploy.Options, result *deployJSON) error { switch a.dags { case deploy.DagsFromImage: - // The Deployment got its DAGs from the image just pushed. A build - // from this project baked them in; an image built elsewhere carries - // whatever it carries, possibly none, and nothing else would say so. - if a.imageName != "" { - warn(result, opts.Progress, fmt.Sprintf(warningImageNameDagsInImage, a.imageName)) - } + // The Deployment got its DAGs from the image just pushed, which was + // built elsewhere and carries whatever it carries, possibly none, and + // nothing else would say so. + warn(result, opts.Progress, fmt.Sprintf(warningImageNameDagsInImage, a.imageName)) return nil case deploy.DagsFromElsewhere: return nil case deploy.DagsFromUpload, deploy.DagsFromUnknown: - // Uploaded below. A Deployment the deploy did not place is uploaded - // to as a deploy always has, and DagsOnlyDeploy's refusals decide. + // Uploaded below, from a pyproject.toml project. A Deployment the + // deploy did not place is uploaded to as a deploy always has, and + // DagsOnlyDeploy's refusals decide. + } + if !a.inProject { + // No project here: the working directory's dags/ belongs to no + // project this deploy knows, so the image ships alone, and an + // upload that a Deployment taking them would want is said not to + // have happened, whatever show_warnings is. + msg := noticeImageAlone + if a.dags == deploy.DagsFromUnknown { + msg = noticeImageAloneUnplaced + } + alwaysWarn(result, opts.Progress, fmt.Sprintf(msg, a.deployment)) + return nil } _, err := DagsOnlyDeploy(a.client, a.workspace, a.deployment, a.path, nil, true, a.description, opts) @@ -285,11 +347,7 @@ func deployDagsAfterImage(a *dagsAfterImage, opts deploy.Options, result *deploy // DagsOnlyDeploy refuses before it looks for the directory, so the // Deployment takes uploads: one was due and did not happen, which is // said whatever show_warnings is. - advice := adviceCreateDagsDir - if a.imageName != "" { - advice = adviceRunFromProject - } - alwaysWarn(result, opts.Progress, fmt.Sprintf(noticeNoDagsDir, a.path, advice)) + alwaysWarn(result, opts.Progress, fmt.Sprintf(noticeNoDagsDir, a.path)) return nil case a.dags == deploy.DagsFromUnknown && errors.Is(err, deploy.ErrAppConfigUnread): // Whether the Deployment takes uploads at all could not be read, diff --git a/cmd/apc/deploy_dags_unix_test.go b/cmd/apc/deploy_dags_unix_test.go index ec9a38cb4..8d4f28959 100644 --- a/cmd/apc/deploy_dags_unix_test.go +++ b/cmd/apc/deploy_dags_unix_test.go @@ -11,7 +11,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/astronomer/astro-cli/config" "github.com/astronomer/astro-cli/internal/platform/apc/deploy" "github.com/astronomer/astro-cli/internal/platform/apc/houston" ) @@ -22,17 +21,14 @@ func TestDeployFailsOnADagsDirectoryItCannotRead(t *testing.T) { if os.Geteuid() == 0 { t.Skip("root ignores the directory modes this failure is injected with") } - project := filepath.Join(t.TempDir(), "project") - require.NoError(t, os.MkdirAll(filepath.Join(project, "dags"), 0o755)) - prev := config.WorkingPath - config.WorkingPath = project - t.Cleanup(func() { config.WorkingPath = prev }) + project := inProject(t, true) deployMocks(t, deploy.Deployed{DeploymentID: "dep-ac", Dags: deploy.DagsFromUpload}, nil) uploads := realDagsOnlyDeploy(t) - require.NoError(t, os.Chmod(project, 0o000)) - t.Cleanup(func() { _ = os.Chmod(project, 0o755) }) - run := runAPC(t, dagsAPI(dagsDeployment(houston.DagOnlyDeploymentType, true), cfgOf(takesDagUploads)), "", "deploy", "dep-ac", "-o", "json") + dags := filepath.Join(project, "dags") + require.NoError(t, os.Chmod(dags, 0o000)) + t.Cleanup(func() { _ = os.Chmod(dags, 0o755) }) + run := runAPC(t, dagsAPI(dagsDeployment(houston.DagOnlyDeploymentType, true), cfgOf(takesDagUploads)), "", "deploy", "dep-ac", "--image-name", "img:1", "-o", "json") assert.NotEqual(t, 0, run.code) require.ErrorIs(t, run.err, syscall.EACCES) assert.NotErrorIs(t, run.err, deploy.ErrNoDagsDirectory) diff --git a/cmd/apc/deploy_imagealone_test.go b/cmd/apc/deploy_imagealone_test.go new file mode 100644 index 000000000..096562018 --- /dev/null +++ b/cmd/apc/deploy_imagealone_test.go @@ -0,0 +1,88 @@ +package apc + +import ( + "fmt" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/astronomer/astro-cli/cmd/cliout" + "github.com/astronomer/astro-cli/config" + "github.com/astronomer/astro-cli/internal/platform/apc/deploy" +) + +// Outside any project --image-name ships the image alone: the working +// directory's dags/ is not uploaded, even to a Deployment that takes DAG +// uploads, and the deploy says the DAGs were not updated, whatever +// show_warnings is. +func TestDeployImageNameOutsideAProjectShipsTheImageAlone(t *testing.T) { + toUpload := deployPushed + toUpload.Dags = deploy.DagsFromUpload + for _, tc := range []struct { + name string + deployed deploy.Deployed + want string + }{ + {"a Deployment that takes DAG uploads", toUpload, fmt.Sprintf(noticeImageAlone, "dep-ac")}, + {"a Deployment not placed", deployPushed, fmt.Sprintf(noticeImageAloneUnplaced, "dep-ac")}, + } { + t.Run(tc.name, func(t *testing.T) { + inWorkingDir(t, true) + seen := deployMocks(t, tc.deployed, nil) + run := runAPC(t, newAPCClient(), "", "deploy", "dep-ac", "--image-name", "img:1", "-o", "json") + require.Equal(t, 0, run.code, "stderr:\n%s", run.stderr) + assert.Zero(t, seen.dagUploads, "no dags/ is uploaded from outside a project") + var got deployJSON + decodeOne(t, run.stdout, &got) + assert.Equal(t, "image", got.Type) + assert.Equal(t, []string{tc.want}, got.Warnings) + }) + } +} + +// Below a pyproject.toml project's root every deploy is refused, --image-name +// included, with the project's root to run from. +func TestDeployBelowAProjectRoot(t *testing.T) { + deployMocks(t, deployPushed, nil) + dir := inProject(t, true) + prev := config.WorkingPath + config.WorkingPath = filepath.Join(dir, "dags") + t.Cleanup(func() { config.WorkingPath = prev }) + run := runAPC(t, newAPCClient(), "", "deploy", "dep-ac", "--image-name", "img:1", "-o", "json") + require.Error(t, run.err) + assert.Contains(t, run.err.Error(), "Run the deploy from the project directory, "+dir) +} + +// The uncommitted-changes gate is for a deploy that reads the checkout. An +// image deployed alone from outside any project reads nothing from the +// directory, so a dirty one does not stop it; a project's deploy is held to +// it as before. +func TestDeployUncommittedChangesGateOnlyForAProject(t *testing.T) { + prev := hasUncommittedChanges + hasUncommittedChanges = func(string) bool { return true } + t.Cleanup(func() { hasUncommittedChanges = prev }) + + inWorkingDir(t, false) + deployMocks(t, deployPushed, nil) + run := runAPC(t, newAPCClient(), "", "deploy", "dep-ac", "--image-name", "img:1", "-o", "json") + require.Equal(t, 0, run.code, "stderr:\n%s", run.stderr) + + inProject(t, true) + deployMocks(t, deployPushed, nil) + run = runAPC(t, newAPCClient(), "", "deploy", "dep-ac", "--image-name", "img:1", "-o", "json") + require.ErrorIs(t, run.err, errUncommittedChanges) +} + +// --dags with --image-name is refused in astro deploy's words, before +// anything is asked: a deploy of DAGs alone has no use for an image. +func TestDeployRefusesDagsWithImageName(t *testing.T) { + inProject(t, true) + seen := deployMocks(t, deployPushed, nil) + run := runAPC(t, newAPCClient(), "", "deploy", "dep-ac", "--dags", "--image-name", "img:1", "-o", "json") + require.Error(t, run.err) + assert.EqualError(t, run.err, "--dags deploys only your DAGs; drop --image-name") + assert.True(t, cliout.IsUsage(run.err)) + assert.Zero(t, seen.dagUploads) +} diff --git a/cmd/apc/deploy_test.go b/cmd/apc/deploy_test.go index 01d399fda..ce9f88970 100644 --- a/cmd/apc/deploy_test.go +++ b/cmd/apc/deploy_test.go @@ -2,12 +2,20 @@ package apc import ( "bytes" - "fmt" + "encoding/json" + "errors" + "os" + "path/filepath" + "testing" - "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/astronomer/astro-cli/cmd/cliout" + "github.com/astronomer/astro-cli/cmd/utils" "github.com/astronomer/astro-cli/internal/platform/apc/deploy" "github.com/astronomer/astro-cli/internal/platform/apc/houston" + "github.com/astronomer/astro-cli/internal/project" testUtil "github.com/astronomer/astro-cli/pkg/testing" ) @@ -23,14 +31,15 @@ func execDeployCmd(args ...string) error { // that did not happen exits non-zero, with no usage block under it. It used to // print a note and return nil: exit 0. --force deploys anyway. func (s *Suite) TestDeployRefusesUncommittedChanges() { + // A project deploy: the gate is for what it reads from the checkout. + inProject(s.T(), true) appConfig = &houston.AppConfig{} - EnsureProjectDir = func(cmd *cobra.Command, args []string) error { return nil } prev := hasUncommittedChanges hasUncommittedChanges = func(string) bool { return true } defer func() { hasUncommittedChanges = prev }() deployed := 0 - DeployAirflowImage = func(houstonClient houston.ClientInterface, path, deploymentID, wsID string, ignoreCacheDeploy, prompt bool, description string, isImageOnlyDeploy bool, imageName string, _ deploy.Options) (deploy.Deployed, error) { + DeployAirflowImage = func(houstonClient houston.ClientInterface, deploymentID, wsID string, prompt, isImageOnlyDeploy bool, imageName string, _ deploy.Options) (deploy.Deployed, error) { deployed++ return deploy.Deployed{DeploymentID: deploymentID}, nil } @@ -44,7 +53,7 @@ func (s *Suite) TestDeployRefusesUncommittedChanges() { var printed bytes.Buffer cmd.SetOut(&printed) cmd.SetErr(&printed) - cmd.SetArgs([]string{"test-deployment-id", "--workspace-id", "test-workspace-id"}) + cmd.SetArgs([]string{"test-deployment-id", "--image-name", "img:1", "--workspace-id", "test-workspace-id"}) restore := testUtil.SetupOSArgsForGinkgo() _, err := cmd.ExecuteC() restore() @@ -52,68 +61,51 @@ func (s *Suite) TestDeployRefusesUncommittedChanges() { s.Equal(0, deployed, "a refused deploy ships nothing") s.NotContains(printed.String(), "Usage:", "the refusal is not a usage mistake") - s.NoError(execDeployCmd("test-deployment-id", "--workspace-id", "test-workspace-id", "--force")) + s.NoError(execDeployCmd("test-deployment-id", "--image-name", "img:1", "--workspace-id", "test-workspace-id", "--force")) s.Equal(1, deployed, "--force deploys anyway") } func (s *Suite) TestDeploy() { + // From a project, whose DAGs follow the image to a Deployment that takes them. + inProject(s.T(), true) appConfig = &houston.AppConfig{ BYORegistryDomain: "test.registry.io", Flags: houston.FeatureFlags{ BYORegistryEnabled: true, }, } - EnsureProjectDir = func(cmd *cobra.Command, args []string) error { - return nil - } - DeployAirflowImage = func(houstonClient houston.ClientInterface, path, deploymentID, wsID string, ignoreCacheDeploy, prompt bool, description string, isImageOnlyDeploy bool, imageName string, _ deploy.Options) (deploy.Deployed, error) { - if description == "" { - return deploy.Deployed{DeploymentID: deploymentID}, fmt.Errorf("description should not be empty") - } + DeployAirflowImage = func(houstonClient houston.ClientInterface, deploymentID, wsID string, prompt, isImageOnlyDeploy bool, imageName string, _ deploy.Options) (deploy.Deployed, error) { return deploy.Deployed{DeploymentID: deploymentID}, nil } + var gotDescription string DagsOnlyDeploy = func(houstonClient houston.ClientInterface, wsID, deploymentID, dagsParentPath string, dagDeployURL *string, cleanUpFiles bool, description string, _ deploy.Options) (string, error) { + gotDescription = description return deploymentID, nil } - err := execDeployCmd([]string{"-f"}...) - s.NoError(err) - - err = execDeployCmd([]string{"-f", "test-deployment-id"}...) - s.NoError(err) - - err = execDeployCmd([]string{"test-deployment-id", "--save"}...) - s.NoError(err) - - // Test when description is provided using the flag --description - err = execDeployCmd([]string{"test-deployment-id", "--description", "Initial deployment", "--force"}...) - s.NoError(err) - - // Test when the default description is used - DeployAirflowImage = func(houstonClient houston.ClientInterface, path, deploymentID, wsID string, ignoreCacheDeploy, prompt bool, description string, isImageOnlyDeploy bool, imageName string, _ deploy.Options) (deploy.Deployed, error) { - expectedDesc := "Deployed via " - if description != expectedDesc { - return deploy.Deployed{DeploymentID: deploymentID}, fmt.Errorf("expected description to be '%s', but got '%s'", expectedDesc, description) - } - return deploy.Deployed{DeploymentID: deploymentID}, nil - } + s.NoError(execDeployCmd("-f", "--image-name", "img:1")) + s.NoError(execDeployCmd("-f", "test-deployment-id", "--image-name", "img:1")) - err = execDeployCmd([]string{"test-deployment-id", "--force"}...) - s.NoError(err) + // The description reaches the DAG upload that follows the image. + s.NoError(execDeployCmd("test-deployment-id", "--image-name", "img:1", "--description", "Initial deployment", "--force")) + s.Equal("Initial deployment", gotDescription) + description = "" + s.NoError(execDeployCmd("test-deployment-id", "--image-name", "img:1", "--force")) + s.Equal("Deployed via ", gotDescription) // Restore DagsOnlyDeploy to default behavior DagsOnlyDeploy = deploy.DagsOnlyDeploy s.Run("error should be returned for astro deploy, if DeployAirflowImage throws error", func() { - DeployAirflowImage = func(houstonClient houston.ClientInterface, path, deploymentID, wsID string, ignoreCacheDeploy, prompt bool, description string, isImageOnlyDeploy bool, imageName string, _ deploy.Options) (deploy.Deployed, error) { + DeployAirflowImage = func(houstonClient houston.ClientInterface, deploymentID, wsID string, prompt, isImageOnlyDeploy bool, imageName string, _ deploy.Options) (deploy.Deployed, error) { return deploy.Deployed{DeploymentID: deploymentID}, deploy.ErrNoWorkspaceID } - err := execDeployCmd([]string{"-f"}...) + err := execDeployCmd("-f", "--image-name", "img:1") s.ErrorIs(err, deploy.ErrNoWorkspaceID) - DeployAirflowImage = func(houstonClient houston.ClientInterface, path, deploymentID, wsID string, ignoreCacheDeploy, prompt bool, description string, isImageOnlyDeploy bool, imageName string, _ deploy.Options) (deploy.Deployed, error) { + DeployAirflowImage = func(houstonClient houston.ClientInterface, deploymentID, wsID string, prompt, isImageOnlyDeploy bool, imageName string, _ deploy.Options) (deploy.Deployed, error) { return deploy.Deployed{DeploymentID: deploymentID}, nil } }) @@ -122,46 +114,48 @@ func (s *Suite) TestDeploy() { DagsOnlyDeploy = func(houstonClient houston.ClientInterface, wsID, deploymentID, dagsParentPath string, dagDeployURL *string, cleanUpFiles bool, description string, _ deploy.Options) (string, error) { return deploymentID, deploy.ErrNoWorkspaceID } - err := execDeployCmd([]string{"-f"}...) + err := execDeployCmd("-f", "--image-name", "img:1") s.ErrorIs(err, deploy.ErrNoWorkspaceID) }) s.Run("Test for the flag --dags when the feature is disabled", func() { + inProject(s.T(), true) DagsOnlyDeploy = func(houstonClient houston.ClientInterface, wsID, deploymentID, dagsParentPath string, dagDeployURL *string, cleanUpFiles bool, description string, _ deploy.Options) (string, error) { return deploymentID, deploy.ErrDagOnlyDeployDisabledInConfig } - err := execDeployCmd([]string{"test-deployment-id", "--dags", "--force"}...) + err := execDeployCmd("test-deployment-id", "--dags", "--force") s.ErrorIs(err, deploy.ErrDagOnlyDeployDisabledInConfig) }) s.Run("Test when both the flags --dags and --image are passed", func() { - err := execDeployCmd([]string{"test-deployment-id", "--dags", "--image", "--force"}...) + inProject(s.T(), true) + err := execDeployCmd("test-deployment-id", "--dags", "--image", "--force") s.ErrorIs(err, ErrBothDagsOnlyAndImageOnlySet) }) s.Run("Test for the flag --image for image deployment", func() { - DeployAirflowImage = func(houstonClient houston.ClientInterface, path, deploymentID, wsID string, ignoreCacheDeploy, prompt bool, description string, isImageOnlyDeploy bool, imageName string, _ deploy.Options) (deploy.Deployed, error) { + DeployAirflowImage = func(houstonClient houston.ClientInterface, deploymentID, wsID string, prompt, isImageOnlyDeploy bool, imageName string, _ deploy.Options) (deploy.Deployed, error) { return deploy.Deployed{DeploymentID: deploymentID}, deploy.ErrDeploymentTypeIncorrectForImageOnly } - err := execDeployCmd([]string{"test-deployment-id", "--image", "--force"}...) + err := execDeployCmd("test-deployment-id", "--image", "--image-name", "img:1", "--force") s.ErrorIs(err, deploy.ErrDeploymentTypeIncorrectForImageOnly) }) s.Run("Test for the flag --image for dags-only deployment", func() { - DeployAirflowImage = func(houstonClient houston.ClientInterface, path, deploymentID, wsID string, ignoreCacheDeploy, prompt bool, description string, isImageOnlyDeploy bool, imageName string, _ deploy.Options) (deploy.Deployed, error) { + DeployAirflowImage = func(houstonClient houston.ClientInterface, deploymentID, wsID string, prompt, isImageOnlyDeploy bool, imageName string, _ deploy.Options) (deploy.Deployed, error) { return deploy.Deployed{DeploymentID: deploymentID}, nil } // This function is not called since --image is passed DagsOnlyDeploy = func(houstonClient houston.ClientInterface, wsID, deploymentID, dagsParentPath string, dagDeployURL *string, cleanUpFiles bool, description string, _ deploy.Options) (string, error) { return deploymentID, deploy.ErrNoWorkspaceID } - err := execDeployCmd([]string{"test-deployment-id", "--image", "--force"}...) + err := execDeployCmd("test-deployment-id", "--image", "--image-name", "img:1", "--force") s.ErrorIs(err, nil) }) s.Run("Test for the flag --image-name", func() { var capturedImageName string - DeployAirflowImage = func(houstonClient houston.ClientInterface, path, deploymentID, wsID string, ignoreCacheDeploy, prompt bool, description string, isImageOnlyDeploy bool, imageName string, _ deploy.Options) (deploy.Deployed, error) { + DeployAirflowImage = func(houstonClient houston.ClientInterface, deploymentID, wsID string, prompt, isImageOnlyDeploy bool, imageName string, _ deploy.Options) (deploy.Deployed, error) { capturedImageName = imageName // Capture the imageName return deploy.Deployed{DeploymentID: deploymentID}, nil } @@ -169,7 +163,7 @@ func (s *Suite) TestDeploy() { return deploymentID, nil } testImageName := "test-image-name" // Set the expected image name - err := execDeployCmd([]string{"test-deployment-id", "--image-name=" + testImageName, "--force", "--workspace-id=" + mockWorkspace.ID}...) + err := execDeployCmd("test-deployment-id", "--image-name="+testImageName, "--force", "--workspace-id="+mockWorkspace.ID) s.ErrorIs(err, nil) s.Equal(testImageName, capturedImageName, "The imageName passed to DeployAirflowImage is incorrect") @@ -183,7 +177,7 @@ func (s *Suite) TestDeploy() { deployAirflowImageCalled := false // Mock function for DeployAirflowImage - DeployAirflowImage = func(houstonClient houston.ClientInterface, path, deploymentID, wsID string, ignoreCacheDeploy, prompt bool, description string, isImageOnlyDeploy bool, imageName string, _ deploy.Options) (deploy.Deployed, error) { + DeployAirflowImage = func(houstonClient houston.ClientInterface, deploymentID, wsID string, prompt, isImageOnlyDeploy bool, imageName string, _ deploy.Options) (deploy.Deployed, error) { deployAirflowImageCalled = true // Set the flag if this function is called return deploy.Deployed{DeploymentID: deploymentID}, nil } @@ -191,7 +185,7 @@ func (s *Suite) TestDeploy() { return deploy.Deployed{}, nil } testImageName := "test-image-name" // Set the expected image name - err := execDeployCmd([]string{"test-deployment-id", "--image-name=" + testImageName, "--force", "--remote", "--workspace-id=" + mockWorkspace.ID}...) + err := execDeployCmd("test-deployment-id", "--image-name="+testImageName, "--force", "--remote", "--workspace-id="+mockWorkspace.ID) s.ErrorIs(err, nil) // Assert that DeployAirflowImage was NOT called s.False(deployAirflowImageCalled, "DeployAirflowImage should not be called when --remote is specified") @@ -202,7 +196,7 @@ func (s *Suite) TestDeploy() { return deploy.Deployed{}, errNoWorkspaceFound } testImageName := "test-image-name" // Set the expected image name - err := execDeployCmd([]string{"test-deployment-id", "--image-name=" + testImageName, "--force", "--remote", "--workspace-id=" + mockWorkspace.ID}...) + err := execDeployCmd("test-deployment-id", "--image-name="+testImageName, "--force", "--remote", "--workspace-id="+mockWorkspace.ID) s.ErrorIs(err, errNoWorkspaceFound) }) @@ -210,7 +204,10 @@ func (s *Suite) TestDeploy() { UpdateDeploymentImage = func(houstonClient houston.ClientInterface, deploymentID, wsID, runtimeVersion, imageName string, _ deploy.Options) (deploy.Deployed, error) { return deploy.Deployed{}, errNoWorkspaceFound } - err := execDeployCmd([]string{"test-deployment-id", "--force", "--remote", "--workspace-id=" + mockWorkspace.ID}...) + err := execDeployCmd("test-deployment-id", "--force", "--remote", "--workspace-id="+mockWorkspace.ID) + s.ErrorIs(err, ErrImageNameNotPassedForRemoteFlag) + // An empty --image-name= names no image either. + err = execDeployCmd("test-deployment-id", "--force", "--remote", "--image-name=", "--workspace-id="+mockWorkspace.ID) s.ErrorIs(err, ErrImageNameNotPassedForRemoteFlag) }) @@ -218,7 +215,96 @@ func (s *Suite) TestDeploy() { DagsOnlyDeploy = func(houstonClient houston.ClientInterface, wsID, deploymentID, dagsParentPath string, dagDeployURL *string, cleanUpFiles bool, description string, _ deploy.Options) (string, error) { return deploymentID, deploy.ErrBYORegistryDomainNotSet } - err := execDeployCmd([]string{"-f"}...) + err := execDeployCmd("-f", "--image-name", "img:1") s.ErrorIs(err, deploy.ErrBYORegistryDomainNotSet) }) } + +// in1xProject points the deploy at a project in the Astro CLI 1.x layout. +func in1xProject(t *testing.T) string { + t.Helper() + dir := inWorkingDir(t, true) + require.NoError(t, os.WriteFile(filepath.Join(dir, "Dockerfile"), []byte("FROM quay.io/astronomer/astro-runtime:12.0.0\n"), 0o600)) + require.NoError(t, os.MkdirAll(filepath.Join(dir, ".astro"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, ".astro", "config.yaml"), []byte("project:\n name: demo\n"), 0o600)) + return dir +} + +// v2 builds no project for APC: a deploy that would build one is refused +// before anything is asked or sent, saying to use Astro CLI 1.x, and a +// project in the 1.x layout makes no deploy at all. --image-name and, from a +// pyproject.toml project, --dags still deploy. +func TestDeployRefusesWhatV2DoesNotDeploy(t *testing.T) { + apc1x := utils.Deploy1xRefusedAPC("this project", "") + for _, tc := range []struct { + name string + setup func(t *testing.T) string + args []string + says string + // usage is whether the refusal is a usage error (exit 2). + usage bool + // noProject is whether it is a no_project failure. + noProject bool + }{ + {"a build outside any project", func(t *testing.T) string { return inWorkingDir(t, true) }, nil, errBuildDeployNoProject, false, true}, + {"a build of a pyproject.toml project", func(t *testing.T) string { return inProject(t, true) }, nil, errBuildDeployManifest, true, false}, + {"an empty --image-name= is a build", func(t *testing.T) string { return inProject(t, true) }, []string{"--image-name="}, errBuildDeployManifest, true, false}, + {"a build of a 1.x project", in1xProject, nil, apc1x, false, true}, + {"--image-name from a 1.x project", in1xProject, []string{"--image-name", "img:1"}, apc1x, false, true}, + {"--dags from a 1.x project", in1xProject, []string{"--dags"}, apc1x, false, true}, + {"--dags outside any project", func(t *testing.T) string { return inWorkingDir(t, true) }, []string{"--dags"}, "this is not an Astro project directory", false, true}, + } { + t.Run(tc.name, func(t *testing.T) { + seen := deployMocks(t, deployPushed, nil) + imageDeploys := 0 + DeployAirflowImage = func(houston.ClientInterface, string, string, bool, bool, string, deploy.Options) (deploy.Deployed, error) { + imageDeploys++ + return deployPushed, nil + } + tc.setup(t) + for _, format := range []string{"text", "json"} { + args := append(append([]string{"deploy", "dep-ac"}, tc.args...), "-o", format) + run := runAPC(t, newAPCClient(), "", args...) + require.Error(t, run.err, format) + assert.Contains(t, run.err.Error(), tc.says, format) + assert.Equal(t, tc.usage, cliout.IsUsage(run.err), format) + var notFound *project.NotFoundError + assert.Equal(t, tc.noProject, errors.As(run.err, ¬Found), format) + assert.Zero(t, imageDeploys, "%s: nothing is deployed", format) + assert.Zero(t, seen.dagUploads, "%s: nothing is uploaded", format) + if format == "json" { + var obj struct { + Error string `json:"error"` + } + require.NoError(t, json.Unmarshal([]byte(run.stdout), &obj), "stdout: %s", run.stdout) + assert.Contains(t, obj.Error, tc.says) + } + } + }) + } +} + +// The deploys v2 keeps on APC need no 1.x project: --image-name from +// anywhere else, and --dags from a pyproject.toml project. +func TestDeployKeepsImageNameAndDags(t *testing.T) { + for _, tc := range []struct { + name string + setup func(t *testing.T) string + args []string + }{ + {"--image-name outside any project", func(t *testing.T) string { return inWorkingDir(t, false) }, []string{"--image-name", "img:1"}}, + {"--image-name from a pyproject.toml project", func(t *testing.T) string { return inProject(t, true) }, []string{"--image-name", "img:1"}}, + {"--image-name --remote outside any project", func(t *testing.T) string { return inWorkingDir(t, false) }, []string{"--image-name", "img:1", "--remote", "--runtime-version", "12.1.1"}}, + {"--dags from a pyproject.toml project", func(t *testing.T) string { return inProject(t, true) }, []string{"--dags"}}, + } { + t.Run(tc.name, func(t *testing.T) { + deployMocks(t, deployPushed, nil) + tc.setup(t) + run := runAPC(t, newAPCClient(), "", append(append([]string{"deploy", "dep-ac"}, tc.args...), "-o", "json")...) + require.Equal(t, 0, run.code, "stderr:\n%s", run.stderr) + var got deployJSON + decodeOne(t, run.stdout, &got) + assert.Equal(t, "dep-ac", got.Deployment) + }) + } +} diff --git a/cmd/apc/deployment_json_test.go b/cmd/apc/deployment_json_test.go index 35f34a458..b51c7bff5 100644 --- a/cmd/apc/deployment_json_test.go +++ b/cmd/apc/deployment_json_test.go @@ -563,13 +563,12 @@ type deployCalls struct { // too; dags is what the DAG deploy returns. func deployMocks(t *testing.T, deployed deploy.Deployed, dags error) *deployCalls { t.Helper() - prevImage, prevDags, prevRemote, prevEnsure := DeployAirflowImage, DagsOnlyDeploy, UpdateDeploymentImage, EnsureProjectDir + prevImage, prevDags, prevRemote := DeployAirflowImage, DagsOnlyDeploy, UpdateDeploymentImage t.Cleanup(func() { - DeployAirflowImage, DagsOnlyDeploy, UpdateDeploymentImage, EnsureProjectDir = prevImage, prevDags, prevRemote, prevEnsure + DeployAirflowImage, DagsOnlyDeploy, UpdateDeploymentImage = prevImage, prevDags, prevRemote }) seen := new(deployCalls) - EnsureProjectDir = func(*cobra.Command, []string) error { return nil } - DeployAirflowImage = func(_ houston.ClientInterface, _, deploymentID, _ string, _, _ bool, _ string, _ bool, _ string, opts deploy.Options) (deploy.Deployed, error) { + DeployAirflowImage = func(_ houston.ClientInterface, deploymentID, _ string, _, _ bool, _ string, opts deploy.Options) (deploy.Deployed, error) { seen.opts = opts fmt.Fprintln(opts.Progress, "Deploying: rel-ac") return deployed, nil @@ -605,15 +604,24 @@ func inWorkingDir(t *testing.T, withDags bool) string { return dir } +// inProject is inWorkingDir with a pyproject.toml project in the directory, +// the one kind of project whose DAGs --dags uploads. +func inProject(t *testing.T, withDags bool) string { + t.Helper() + dir := inWorkingDir(t, withDags) + require.NoError(t, os.WriteFile(filepath.Join(dir, "pyproject.toml"), []byte("[project]\nname = \"demo\"\n\n[tool.astro]\n"), 0o600)) + return dir +} + // warningsOff turns show_warnings off for the deploy's run. runAPC resets // the config before it runs the tree, so this is done from the first // platform call. func warningsOff(t *testing.T) { t.Helper() image := DeployAirflowImage - DeployAirflowImage = func(c houston.ClientInterface, path, deploymentID, wsID string, noCache, prompt bool, description string, imageOnly bool, imageName string, opts deploy.Options) (deploy.Deployed, error) { + DeployAirflowImage = func(c houston.ClientInterface, deploymentID, wsID string, prompt, imageOnly bool, imageName string, opts deploy.Options) (deploy.Deployed, error) { require.NoError(t, config.CFG.ShowWarnings.SetHomeString("false")) - return image(c, path, deploymentID, wsID, noCache, prompt, description, imageOnly, imageName, opts) + return image(c, deploymentID, wsID, prompt, imageOnly, imageName, opts) } remote := UpdateDeploymentImage UpdateDeploymentImage = func(c houston.ClientInterface, deploymentID, wsID, runtimeVersion, imageName string, opts deploy.Options) (deploy.Deployed, error) { @@ -685,14 +693,9 @@ func inImageWarning(image string) string { } // noDagsNotice is the notice for a DAG upload skipped for want of a dags -// directory in dir: --image-name may run from anywhere, and a deploy -// without it runs from a project, which lacks one. -func noDagsNotice(dir string, imageName bool) string { - advice := "To upload Dags, create a dags directory in the project." - if imageName { - advice = "To upload them, run the deploy from the project directory." - } - return "no Dags were uploaded: there is no dags directory in " + dir + ", and the Deployment keeps the Dags it had. " + advice +// directory in the pyproject.toml project at dir. +func noDagsNotice(dir string) string { + return "no Dags were uploaded: there is no dags directory in " + dir + ", and the Deployment keeps the Dags it had. To upload Dags, create a dags directory in the project." } // deployPushed is what the image deploy reports in these tests. @@ -702,8 +705,9 @@ func TestDeployJSON(t *testing.T) { pushed := deployPushed t.Run("image and dags", func(t *testing.T) { + inProject(t, true) deployMocks(t, pushed, nil) - run := runAPC(t, newAPCClient(), "", "deploy", "dep-ac", "-o", "json") + run := runAPC(t, newAPCClient(), "", "deploy", "dep-ac", "--image-name", "img:1", "-o", "json") require.Equal(t, 0, run.code, "stderr:\n%s", run.stderr) var got deployJSON decodeOne(t, run.stdout, &got) @@ -712,8 +716,9 @@ func TestDeployJSON(t *testing.T) { }) t.Run("a Deployment that takes no DAG-only deploy is an image deploy", func(t *testing.T) { + inProject(t, true) deployMocks(t, pushed, deploy.ErrDagOnlyDeployNotEnabledForDeployment) - run := runAPC(t, newAPCClient(), "", "deploy", "dep-ac", "-o", "json") + run := runAPC(t, newAPCClient(), "", "deploy", "dep-ac", "--image-name", "img:1", "-o", "json") require.Equal(t, 0, run.code, "stderr:\n%s", run.stderr) var got deployJSON decodeOne(t, run.stdout, &got) @@ -774,9 +779,8 @@ func TestDeployJSON(t *testing.T) { assert.NotContains(t, run.stderr, "Warning") }) - // No warning where it would mislead: a build from the project baked its - // DAGs in, a git-sync or volume Deployment gets them from elsewhere, and - // --image says it deploys no DAGs itself. + // No warning where it would mislead: a git-sync or volume Deployment gets + // its DAGs from elsewhere, and --image says it deploys no DAGs itself. toGitSync := pushed toGitSync.Dags = deploy.DagsFromElsewhere for _, tc := range []struct { @@ -785,7 +789,6 @@ func TestDeployJSON(t *testing.T) { dags error args []string }{ - {"a Dockerfile build to an image Deployment", toImage, refused, []string{"deploy", "dep-ac", "-o", "json"}}, {"--image-name to a git-sync Deployment", toGitSync, refused, []string{"deploy", "dep-ac", "--image-name", "img:1", "-o", "json"}}, {"--image-name --remote to a git-sync Deployment", toGitSync, refused, []string{"deploy", "dep-ac", "--image-name", "img:1", "--remote", "--runtime-version", "12.1.1", "-o", "json"}}, {"--image --image-name to an image Deployment", toImage, errors.New("must not deploy DAGs"), []string{"deploy", "dep-ac", "--image", "--image-name", "img:1", "-o", "json"}}, @@ -806,19 +809,16 @@ func TestDeployJSON(t *testing.T) { // With no dags directory to upload, a Deployment that takes DAG uploads // keeps the DAGs it has: DagsOnlyDeploy refuses the empty upload that // would delete them. The deploy says so whatever show_warnings is, as it - // is something asked for that did not happen. --image-name skips the - // project check, so it is the usual way to get here, but not the only - // one. + // is something asked for that did not happen. toUpload := pushed toUpload.Dags = deploy.DagsFromUpload t.Run("no dags directory skips the DAG upload, and says so", func(t *testing.T) { - dir := inWorkingDir(t, false) + dir := inProject(t, false) for _, args := range [][]string{ {"deploy", "dep-ac", "--image-name", "img:1", "-o", "json"}, {"deploy", "dep-ac", "--image-name", "img:1", "--remote", "--runtime-version", "12.1.1", "-o", "json"}, - {"deploy", "dep-ac", "-o", "json"}, } { - noDags := noDags(dir, args[2] == "--image-name") + noDags := noDags(dir) for _, quiet := range []bool{false, true} { deployMocks(t, toUpload, nil) uploads := realDagsOnlyDeploy(t) @@ -841,7 +841,7 @@ func TestDeployJSON(t *testing.T) { warningsOff(t) run := runAPC(t, dagsAPI(dagsDeployment(houston.DagOnlyDeploymentType, true), cfgOf(takesDagUploads)), "", "deploy", "dep-ac", "--image-name", "img:1") require.Equal(t, 0, run.code, "stderr:\n%s", run.stderr) - assert.Equal(t, "Deploying: rel-ac\nWarning: "+noDags(dir, true)+"\n", run.stdout, "text: on stdout, once, show_warnings off too") + assert.Equal(t, "Deploying: rel-ac\nWarning: "+noDags(dir)+"\n", run.stdout, "text: on stdout, once, show_warnings off too") }) // A Deployment the image deploy did not place (the zero value) is @@ -850,7 +850,7 @@ func TestDeployJSON(t *testing.T) { // a Houston before 0.29.0) says nothing, as before; one that takes the // upload says it was skipped, as a placed one does. t.Run("a Deployment not placed is uploaded to as before", func(t *testing.T) { - inWorkingDir(t, true) + inProject(t, true) seen := deployMocks(t, pushed, nil) run := runAPC(t, newAPCClient(), "", "deploy", "dep-ac", "--image-name", "img:1", "-o", "json") require.Equal(t, 0, run.code, "stderr:\n%s", run.stderr) @@ -862,7 +862,7 @@ func TestDeployJSON(t *testing.T) { for _, refusal := range []error{deploy.ErrDagOnlyDeployNotEnabledForDeployment, deploy.ErrDagOnlyDeployDisabledInConfig} { seen := deployMocks(t, pushed, refusal) - run := runAPC(t, newAPCClient(), "", "deploy", "dep-ac", "-o", "json") + run := runAPC(t, newAPCClient(), "", "deploy", "dep-ac", "--image-name", "img:1", "-o", "json") require.Equal(t, 0, run.code, "%v stderr:\n%s", refusal, run.stderr) assert.Equal(t, 1, seen.dagUploads, "%v", refusal) var got deployJSON @@ -871,7 +871,7 @@ func TestDeployJSON(t *testing.T) { assert.Empty(t, got.Warnings, "%v", refusal) } - dir := inWorkingDir(t, false) + dir := inProject(t, false) for _, tc := range []struct { name string api func() *mocks.ClientInterface @@ -885,7 +885,7 @@ func TestDeployJSON(t *testing.T) { }, nil}, {"a DAG-only Deployment", func() *mocks.ClientInterface { return dagsAPI(dagsDeployment(houston.DagOnlyDeploymentType, true), cfgOf(takesDagUploads)) - }, []string{noDags(dir, false)}}, + }, []string{noDags(dir)}}, } { for _, quiet := range []bool{false, true} { deployMocks(t, pushed, nil) @@ -893,7 +893,7 @@ func TestDeployJSON(t *testing.T) { if quiet { warningsOff(t) } - run := runAPC(t, tc.api(), "", "deploy", "dep-ac", "-o", "json") + run := runAPC(t, tc.api(), "", "deploy", "dep-ac", "--image-name", "img:1", "-o", "json") require.Equal(t, 0, run.code, "%s quiet=%v stderr:\n%s", tc.name, quiet, run.stderr) assert.Equal(t, 1, *uploads, tc.name) var got deployJSON @@ -908,7 +908,7 @@ func TestDeployJSON(t *testing.T) { }) t.Run("a dags directory is uploaded as before", func(t *testing.T) { - inWorkingDir(t, true) + inProject(t, true) seen := deployMocks(t, toUpload, nil) run := runAPC(t, newAPCClient(), "", "deploy", "dep-ac", "--image-name", "img:1", "-o", "json") require.Equal(t, 0, run.code, "stderr:\n%s", run.stderr) @@ -921,7 +921,7 @@ func TestDeployJSON(t *testing.T) { t.Run("--image", func(t *testing.T) { deployMocks(t, pushed, errors.New("must not deploy DAGs")) - run := runAPC(t, newAPCClient(), "", "deploy", "dep-ac", "--image", "-o", "json") + run := runAPC(t, newAPCClient(), "", "deploy", "dep-ac", "--image", "--image-name", "img:1", "-o", "json") require.Equal(t, 0, run.code, "stderr:\n%s", run.stderr) var got deployJSON decodeOne(t, run.stdout, &got) @@ -932,6 +932,7 @@ func TestDeployJSON(t *testing.T) { t.Run("--dags", func(t *testing.T) { deployMocks(t, pushed, nil) houstonVersion = "1.0.0" + inProject(t, true) run := runAPC(t, newAPCClient(), "", "deploy", "dep-ac", "--dags", "-o", "json") require.Equal(t, 0, run.code, "stderr:\n%s", run.stderr) var got deployJSON @@ -944,6 +945,7 @@ func TestDeployJSON(t *testing.T) { t.Run("--dags with no argument publishes the Deployment it resolved", func(t *testing.T) { deployMocks(t, pushed, nil) houstonVersion = "1.0.0" + inProject(t, true) run := runAPC(t, newAPCClient(), "", "deploy", "--dags", "-o", "json") require.Equal(t, 0, run.code, "stderr:\n%s", run.stderr) var got deployJSON @@ -953,16 +955,17 @@ func TestDeployJSON(t *testing.T) { t.Run("--yes answers the deploy's confirmations", func(t *testing.T) { seen := deployMocks(t, pushed, nil) - run := runAPC(t, newAPCClient(), "", "deploy", "dep-ac", "--yes", "-o", "json") + run := runAPC(t, newAPCClient(), "", "deploy", "dep-ac", "--image-name", "img:1", "--yes", "-o", "json") require.Equal(t, 0, run.code, "stderr:\n%s", run.stderr) assert.True(t, seen.opts.Yes) - run = runAPC(t, newAPCClient(), "", "deploy", "dep-ac", "-o", "json") + run = runAPC(t, newAPCClient(), "", "deploy", "dep-ac", "--image-name", "img:1", "-o", "json") require.Equal(t, 0, run.code) assert.False(t, seen.opts.Yes, "only when given") }) t.Run("--image-name --remote", func(t *testing.T) { + inProject(t, true) deployMocks(t, pushed, nil) run := runAPC(t, newAPCClient(), "", "deploy", "dep-ac", "--image-name", "my/image:1", "--remote", "--runtime-version", "12.1.1", "-o", "json") require.Equal(t, 0, run.code, "stderr:\n%s", run.stderr) @@ -973,8 +976,9 @@ func TestDeployJSON(t *testing.T) { }) t.Run("text prints only the deploy's own progress", func(t *testing.T) { + inProject(t, true) deployMocks(t, pushed, nil) - run := runAPC(t, newAPCClient(), "", "deploy", "dep-ac") + run := runAPC(t, newAPCClient(), "", "deploy", "dep-ac", "--image-name", "img:1") require.Equal(t, 0, run.code, "stderr:\n%s", run.stderr) assert.Equal(t, "Deploying: rel-ac\n", run.stdout) }) @@ -990,9 +994,9 @@ func strp(s string) *string { return &s } func TestDeployDagsNoDagsDirectoryJSON(t *testing.T) { toUpload := deployPushed toUpload.Dags = deploy.DagsFromUpload - dir := inWorkingDir(t, false) + dir := inProject(t, false) for _, args := range [][]string{ - {"deploy", "dep-ac", "--dags", "--image-name", "img:1", "-o", "json"}, + {"deploy", "dep-ac", "--dags", "-o", "json"}, {"deploy", "dep-ac", "--dags"}, } { deployMocks(t, toUpload, nil) @@ -1014,7 +1018,7 @@ func TestDeployDagsNoDagsDirectoryJSON(t *testing.T) { } { deployMocks(t, toUpload, nil) realDagsOnlyDeploy(t) - run := runAPC(t, tc.api, "", "deploy", "dep-ac", "--dags", "--image-name", "img:1", "-o", "json") + run := runAPC(t, tc.api, "", "deploy", "dep-ac", "--dags", "-o", "json") assert.NotEqual(t, 0, run.code, tc.name) require.ErrorIs(t, run.err, tc.want, tc.name) assert.NotErrorIs(t, run.err, deploy.ErrNoDagsDirectory, tc.name) @@ -1046,7 +1050,7 @@ func TestDeployRemoteDagsJSON(t *testing.T) { {"git-sync says nothing", dagsDeployment(houston.GitSyncDeploymentType, true), nil, 0, nil, nil}, // A cluster config it cannot read leaves a DAG-only Deployment // not placed: DagsOnlyDeploy reads the config again, and decides. - {"dag_deploy whose cluster config is read the second time skips the upload, and says so", dagsDeployment(houston.DagOnlyDeploymentType, true), []func() (*houston.AppConfig, error){cfgFails(down), cfgOf(takesDagUploads)}, 1, []string{noDagsNotice("", true)}, []string{noDagsNotice("", true)}}, + {"dag_deploy whose cluster config is read the second time skips the upload, and says so", dagsDeployment(houston.DagOnlyDeploymentType, true), []func() (*houston.AppConfig, error){cfgFails(down), cfgOf(takesDagUploads)}, 1, []string{noDagsNotice("")}, []string{noDagsNotice("")}}, {"dag_deploy whose cluster config is read the second time and refuses says nothing", dagsDeployment(houston.DagOnlyDeploymentType, true), []func() (*houston.AppConfig, error){cfgFails(down), cfgOf(&houston.AppConfig{Version: "2.0.0"})}, 1, nil, nil}, // Read neither time: the image update stands, and the deploy says // the DAGs were not updated, whatever show_warnings is. @@ -1054,7 +1058,7 @@ func TestDeployRemoteDagsJSON(t *testing.T) { } { for _, quiet := range []bool{false, true} { t.Run(fmt.Sprintf("%s quiet=%v", tc.name, quiet), func(t *testing.T) { - dir := inWorkingDir(t, false) + dir := inProject(t, false) deployMocks(t, toUpload, nil) uploads := realDagsOnlyDeploy(t) UpdateDeploymentImage = deploy.UpdateDeploymentImage @@ -1079,8 +1083,8 @@ func TestDeployRemoteDagsJSON(t *testing.T) { want = append([]string(nil), tc.quiet...) } for i, w := range want { - if w == noDagsNotice("", true) { - want[i] = noDagsNotice(dir, true) + if w == noDagsNotice("") { + want[i] = noDagsNotice(dir) } } assert.Equal(t, want, got.Warnings) diff --git a/cmd/apc/output_binding_test.go b/cmd/apc/output_binding_test.go index 203fa9bf6..da5dac0a8 100644 --- a/cmd/apc/output_binding_test.go +++ b/cmd/apc/output_binding_test.go @@ -77,7 +77,7 @@ func TestJSONResultReachesStdoutUnderAnUnboundRoot(t *testing.T) { t.Run("deploy", func(t *testing.T) { deployMocks(t, deploy.Deployed{DeploymentID: "dep-ac", Image: "registry/rel-ac/airflow:deploy-2"}, nil) - stdout, stderr, err := execUnboundRoot(t, newAPCClient(), "deploy", "dep-ac", "-o", "json") + stdout, stderr, err := execUnboundRoot(t, newAPCClient(), "deploy", "dep-ac", "--image-name", "img:1", "-o", "json") require.NoError(t, err, "stderr:\n%s", stderr) var got deployJSON decodeOne(t, stdout, &got) diff --git a/cmd/astro/dbt.go b/cmd/astro/dbt.go index ea08dbf32..4cb758bc3 100644 --- a/cmd/astro/dbt.go +++ b/cmd/astro/dbt.go @@ -12,6 +12,7 @@ import ( "gopkg.in/yaml.v2" "github.com/astronomer/astro-cli/cmd/cliout" + "github.com/astronomer/astro-cli/cmd/utils" "github.com/astronomer/astro-cli/config" "github.com/astronomer/astro-cli/internal/platform/astro/clients/astrov1" astrodeploy "github.com/astronomer/astro-cli/internal/platform/astro/deploy" @@ -119,20 +120,14 @@ func deployDbt(cmd *cobra.Command, args []string) error { dbtProjectPath = config.WorkingPath } - // check that the dbt project path is not within an Astro project - withinAstroProject, err := config.IsWithinProjectDir(dbtProjectPath) - if err != nil { - return fmt.Errorf("failed to verify dbt project path is not within an Astro project: %w", err) - } - if !withinAstroProject { - withinAstroProject = isWithinManifestProject(dbtProjectPath) - } - if withinAstroProject { + // check that the dbt project path is not within an Astro project, by the + // walk every deploy decides by (astro deploy --non-dags checks the same) + if !utils.Locate(dbtProjectPath).None() { return fmt.Errorf("dbt project is within an Astro project. Use 'astro deploy' to deploy your Astro project") } // check that there is a valid dbt project at the dbt project path - err = validateDbtProjectExists(dbtProjectPath) + err := validateDbtProjectExists(dbtProjectPath) if err != nil { return err } diff --git a/cmd/astro/dbt_render.go b/cmd/astro/dbt_render.go index ee8b651a8..40eed0d0a 100644 --- a/cmd/astro/dbt_render.go +++ b/cmd/astro/dbt_render.go @@ -98,7 +98,7 @@ type remoteRuntimeCheckJSON struct { } func newDbtDeployJSON(res *astrodeploy.BundleDeploy, project, projectPath string, waited bool, waitErr error) dbtDeployJSON { - obj := dbtDeployJSON{ + return dbtDeployJSON{ Deployment: res.DeploymentID, DeploymentName: res.DeploymentName, Workspace: res.WorkspaceID, @@ -109,11 +109,17 @@ func newDbtDeployJSON(res *astrodeploy.BundleDeploy, project, projectPath string BundleVersion: res.BundleVersion, Waited: waited, WaitError: errText(waitErr), + Git: bundleGitJSON(res.Git), } - if g := res.Git; g != nil { - obj.Git = &deployGitJSON{CommitSHA: g.CommitSHA, Branch: g.Branch, CommitURL: g.CommitURL} +} + +// bundleGitJSON is the commit a bundle deploy recorded, as a deploy result +// publishes it; nil when it recorded none. +func bundleGitJSON(g *astrodeploy.BundleGit) *deployGitJSON { + if g == nil { + return nil } - return obj + return &deployGitJSON{CommitSHA: g.CommitSHA, Branch: g.Branch, CommitURL: g.CommitURL} } // renderBundleUploaded is the text a finished bundle upload has always @@ -136,11 +142,19 @@ func renderBundleUploaded(version string) func(io.Writer) error { // why, because the upload happened and a script needs its deploy id and // version, and then exits 1 (failedAfterResult). func publishThenWait(cmd *cobra.Command, format cliout.Format, wait bool, deploymentID string, waitTime time.Duration, publish func(waitErr error) error) error { + return publishThenWaitWith(cmd, format, wait, func() error { + return waitForBundle(cmd.ErrOrStderr(), deploymentID, waitTime, astroV1Client) + }, publish) +} + +// publishThenWaitWith is publishThenWait with the wait its caller's, for a +// Deployment reached under a login other than the current context's. +func publishThenWaitWith(cmd *cobra.Command, format cliout.Format, wait bool, waitFor func() error, publish func(waitErr error) error) error { waitDone := func() error { if !wait { return nil } - return waitForBundle(cmd.ErrOrStderr(), deploymentID, waitTime, astroV1Client) + return waitFor() } if format == cliout.FormatJSON { waitErr := waitDone() diff --git a/cmd/astro/dbt_test.go b/cmd/astro/dbt_test.go index 0aea5f5bb..9c23bb88a 100644 --- a/cmd/astro/dbt_test.go +++ b/cmd/astro/dbt_test.go @@ -138,14 +138,26 @@ func (s *DbtSuite) TestDbtDeploy_CustomMountPath() { assert.NoError(s.T(), err) } +// A dbt project in or below a 1.x project, by the walk every deploy decides +// by (a Dockerfile beside .astro, with or without .astro/config.yaml), is +// refused, as astro deploy --non-dags refuses the same bundle path. func (s *DbtSuite) TestDbtDeploy_WithinAstroProject() { - projectDir, cleanup, err := config.CreateTempProject() - assert.NoError(s.T(), err) - defer cleanup() - - err = testExecCmd(newDbtDeployCmd(), "test-deployment-id", "--project-path", projectDir) - assert.Error(s.T(), err) - assert.Contains(s.T(), err.Error(), "dbt project is within an Astro project") + projectDir := s.T().TempDir() + s.Require().NoError(os.WriteFile(filepath.Join(projectDir, "Dockerfile"), []byte("FROM x\n"), 0o600)) + s.Require().NoError(os.MkdirAll(filepath.Join(projectDir, ".astro"), 0o755)) + dbtDir := filepath.Join(projectDir, "include", "dbt") + s.Require().NoError(os.MkdirAll(dbtDir, 0o755)) + + for _, path := range []string{projectDir, dbtDir} { + err := testExecCmd(newDbtDeployCmd(), "test-deployment-id", "--project-path", path) + s.Require().Error(err, path) + s.Contains(err.Error(), "dbt project is within an Astro project", path) + + resetDeployFlagVars() + nonDags := testExecCmd(NewDeployCmd(), "test-deployment-id", "--non-dags", "--non-dags-mount-path", "/x", "--non-dags-local-path", path) + s.Require().Error(nonDags, path) + s.Contains(nonDags.Error(), "within an Astro project", "dbt deploy and --non-dags agree: %s", path) + } } // A project with a pyproject.toml carries no .astro/config.yaml, so the 1.x walk answers false at diff --git a/cmd/astro/deploy.go b/cmd/astro/deploy.go index 2894d4b49..2d78b4504 100644 --- a/cmd/astro/deploy.go +++ b/cmd/astro/deploy.go @@ -2,12 +2,12 @@ package astro import ( "bufio" + "cmp" "context" goerrors "errors" "fmt" "io" "os" - "path/filepath" "time" "github.com/pkg/errors" @@ -23,10 +23,8 @@ import ( astrov1 "github.com/astronomer/astro-cli/internal/platform/astro/clients/astrov1" astrodeploy "github.com/astronomer/astro-cli/internal/platform/astro/deploy" "github.com/astronomer/astro-cli/internal/platform/astro/deployment" - "github.com/astronomer/astro-cli/internal/project" "github.com/astronomer/astro-cli/internal/runtimecatalog" "github.com/astronomer/astro-cli/internal/userstate" - "github.com/astronomer/astro-cli/pkg/git" "github.com/astronomer/astro-cli/pkg/httputil" "github.com/astronomer/astro-cli/pkg/input" "github.com/astronomer/astro-cli/pkg/instances" @@ -37,23 +35,13 @@ import ( ) var ( - forceDeploy bool - forcePrompt bool - saveDeployConfig bool - pytest bool - parse bool dags bool waitForDeploy bool waitTime time.Duration image bool - dagsPath string - pytestFile string - envFile string imageName string - deploymentName string deployDescription string noDagsBaseDir bool - dagBundleName string nonDags bool nonDagsMountPath string nonDagsBundleType string @@ -68,16 +56,12 @@ var ( astro deploy # Deploy only the DAGs - astro deploy --dags` + astro deploy --dags - DeployImage = astrodeploy.Deploy - EnsureProjectDir = utils.EnsureProjectDir - buildSecrets = []string{} - // hasUncommittedChanges is a variable so a test does not depend on the - // state of the checkout it runs in. - hasUncommittedChanges = git.HasUncommittedChanges + # Deploy an image built on this machine, from any directory + astro deploy --image-name ` - errUncommittedChanges = errors.New("project directory has uncommitted changes: commit them, or use `astro deploy [deployment-id] --force` to deploy anyway") + buildSecrets = []string{} ) const ( @@ -85,62 +69,42 @@ const ( imageNameFlag = "image-name" nonDagsFlag = "non-dags" - dagsPathFlag = "dags-path" ) func NewDeployCmd() *cobra.Command { cmd := &cobra.Command{ - Use: "deploy [DEPLOYMENT_ID]", - Short: "Ship this project's code to a Deployment", - Long: "Deploy your project to a Deployment on Astro. This command bundles your project files into a Docker image and pushes that Docker image to Astronomer. In Deployments with Remote Execution enabled, this only updates the Orchestration Plane components (the API Server and Scheduler). For all other components, use `astro remote deploy` instead. It does not include any metadata associated with your local Airflow environment.", - Args: cobra.MaximumNArgs(1), - PreRunE: func(cmd *cobra.Command, args []string) error { - if cmd.Flags().Changed(imageNameFlag) || cmd.Flags().Changed(nonDagsFlag) { - return nil - } - // A project with a pyproject.toml has no .astro/config.yaml, so the 1.x EnsureProjectDir - // check would reject it. The manifest path loads and validates the manifest - // itself, so skip the 1.x check and let deploy() route. - if project.HasManifest(config.WorkingPath) { - return nil - } - // A DAG-only deploy sourcing its DAGs from --dags-path does not read anything - // else from the working directory, unless --pytest/--parse is also used, which - // builds and runs a local image to test-parse the DAGs and so still needs the - // project root. Check the bound values (not cmd.Flags().Changed) so --dags=false - // or an explicit empty --dags-path cannot be misread as enabling the bypass. - if dags && dagsPath != "" && !pytest && !parse { - return nil - } - return EnsureProjectDir(cmd, args) - }, + Use: "deploy [DEPLOYMENT_ID]", + Short: "Ship this project's code to a Deployment", + Long: "Deploy your project to a Deployment on Astro. Run it in a project with a pyproject.toml ([tool.astro]): it builds the project into a Docker image and pushes that image, with the project's DAGs, to Astronomer. A project in the Astro CLI 1.x layout (a Dockerfile and .astro/config.yaml) is converted with astro init, or deployed with Astro CLI 1.x. With --image-name it deploys an image built on this machine, from any directory. In Deployments with Remote Execution enabled, this only updates the Orchestration Plane components (the API Server and Scheduler). For all other components, use astro remote deploy instead. It does not include any metadata associated with your local Airflow environment.", + Args: cobra.MaximumNArgs(1), RunE: deploy, Example: deployExample, } - cmd.Flags().BoolVarP(&forceDeploy, "force", "f", false, "Force deploy even if project contains errors or uncommitted changes") - cmd.Flags().BoolVarP(&forcePrompt, "prompt", "p", false, "Force prompt to choose target deployment") - cmd.Flags().BoolVarP(&saveDeployConfig, "save", "s", false, "Save deployment in config for future deploys") + // --force answered 1.x's checks (uncommitted changes, an empty dags + // directory), which v2's deploy does not make. It stays, accepted and + // read by nothing, because astronomer/deploy-action passes it on every + // deploy (TestDeployManifestDeployActionInvocations); hidden, as there is + // nothing for anyone to choose. + cmd.Flags().BoolP("force", "f", false, "Has no effect; accepted so scripts written for Astro CLI 1.x keep working") + cmd.Flags().MarkHidden("force") //nolint:errcheck // the flag is defined just above + // --prompt asked 1.x to offer the Deployment list even with one named. A + // deploy here asks whenever none is named, so it is accepted, hidden and + // read by nothing, as a pyproject.toml project's deploy always took it. + cmd.Flags().BoolP("prompt", "p", false, "Has no effect; accepted so scripts written for Astro CLI 1.x keep working") + cmd.Flags().MarkHidden("prompt") //nolint:errcheck // the flag is defined just above cmd.Flags().StringVar(&workspaceID, "workspace-id", "", "Workspace for your Deployment") - cmd.Flags().BoolVar(&pytest, "pytest", false, "Deploy code to Astro only if the specified Pytests are passed") - cmd.Flags().StringVarP(&envFile, "env", "e", ".env", "Location of file containing environment variables for Pytests") - cmd.Flags().StringVarP(&pytestFile, "test", "t", "", "Location of Pytests or specific Pytest file. All Pytest files must be located in the tests directory") - cmd.Flags().StringVarP(&imageName, imageNameFlag, "i", "", "Name of a custom image to deploy, or image name with custom tag when used with --client") + cmd.Flags().StringVarP(&imageName, imageNameFlag, "i", "", "Name of a prebuilt local image to deploy instead of building one. Outside a pyproject.toml project it deploys the image alone") cmd.Flags().BoolVarP(&dags, "dags", "d", false, "Push only Dags to your Astro Deployment") cmd.Flags().BoolVar(&noDagsBaseDir, "no-dags-base-dir", false, "Exclude the dags directory prefix from the bundle. Use for Airflow 3.x deployments where sys.path includes the bundle root") - cmd.Flags().StringVar(&dagBundleName, "dag-bundle-name", "", "Deploy Dags to a named Dag bundle on the Deployment instead of the default bundle. Requires Airflow 3, and the bundle must already exist on the Deployment") cmd.Flags().BoolVarP(&image, "image", "", false, "Push only an image to your Astro Deployment; with Dag Deploy enabled, its Dags are not affected") - cmd.Flags().StringVar(&dagsPath, dagsPathFlag, "", "If set deploy dags from this path instead of the dags from working directory") - cmd.Flags().StringVarP(&deploymentName, "deployment-name", "n", "", "Name of the deployment to deploy to") - cmd.Flags().BoolVar(&parse, "parse", false, "Succeed only if all Dags in your Astro project parse without errors") cmd.Flags().BoolVarP(&waitForDeploy, "wait", "w", false, "Wait for the Deployment to become healthy before ending the command") cmd.Flags().DurationVar(&waitTime, "wait-time", deployWaitTime, "Wait time for the Deployment to become healthy before ending the command. Can only be used with --wait=true") - cmd.Flags().MarkHidden("dags-path") //nolint:errcheck // the flag is defined just above; this only errors on an unknown flag name // No -d shorthand: on this command -d has meant --dags since v1, and moving // it would turn `astro deploy -d` from a DAG-only deploy into a target // selector. -d/--deployment is the spelling everywhere the letter is free. - cmd.Flags().StringVar(&manifestDeployment, "deployment", "", "Deployment to deploy to: a link name from the manifest, or a Deployment id. In a project with a pyproject.toml ([tool.astro])") + cmd.Flags().StringVar(&manifestDeployment, "deployment", "", "Deployment to deploy to: a link name from the manifest, or a Deployment id") cmd.Flags().StringVar(&manifestWorkspace, "workspace", "", "Workspace for the deploy, overriding the context") - cmd.Flags().StringVar(&deployOutput, "output", string(cliout.FormatText), "Output format in a project with a pyproject.toml ([tool.astro]): text or json") + cmd.Flags().StringVar(&deployOutput, "output", string(cliout.FormatText), "Output format: text or json") cmd.Flags().StringVarP(&deployDescription, "description", "", "", "Add a description for more context on this deploy") utils.AddBuildSecretFlag(cmd.Flags(), &buildSecrets) cmd.Flags().BoolVar(&nonDags, nonDagsFlag, false, "Deploy a non-Dag bundle from a separate directory, instead of your Astro project. Requires --non-dags-mount-path") @@ -153,12 +117,6 @@ func NewDeployCmd() *cobra.Command { annotateDeployFlag(cmd, "build-secret", deployGroupImage) annotateDeployFlag(cmd, "dags", deployGroupDAG) annotateDeployFlag(cmd, "no-dags-base-dir", deployGroupDAG) - annotateDeployFlag(cmd, "dag-bundle-name", deployGroupDAG) - annotateDeployFlag(cmd, dagsPathFlag, deployGroupDAG) - annotateDeployFlag(cmd, "pytest", deployGroupTest) - annotateDeployFlag(cmd, "test", deployGroupTest) - annotateDeployFlag(cmd, "env", deployGroupTest) - annotateDeployFlag(cmd, "parse", deployGroupTest) annotateDeployFlag(cmd, nonDagsFlag, deployGroupNonDAG) annotateDeployFlag(cmd, "non-dags-mount-path", deployGroupNonDAG) annotateDeployFlag(cmd, "non-dags-bundle-type", deployGroupNonDAG) @@ -169,176 +127,130 @@ func NewDeployCmd() *cobra.Command { return cmd } -func deployTests(parse, pytest, forceDeploy bool, pytestFile string) string { - if pytest && pytestFile == "" { - pytestFile = "all-tests" - } - - if !parse && !pytest && !forceDeploy || parse && !pytest && !forceDeploy || parse && !pytest && forceDeploy { - pytestFile = "parse" - } - - if parse && pytest { - pytestFile = "parse-and-all-tests" - } - - return pytestFile -} - -// manifestDeployIgnores are the flags astro deploy accepts that the manifest path never -// reads, mapped to what to do instead. Each was built for the 1.x deploy, and -// on the manifest path nothing carries it: not the Request built below, and not -// internal/deploy, which contains no reference to any of them. +// deploy ships a project with a pyproject.toml ([tool.astro]), the one kind +// of project v2 deploys (deployManifest), run from the project's root. // -// Refusing beats ignoring. A deploy that quietly skipped --pytest is a deploy -// someone believes ran their tests, and the flag having no effect is exactly -// the thing they cannot see. Porting them is not done yet. +// --non-dags deploys a directory that must not be inside a project at all, so +// where it runs from says nothing about the project. It is checked first. // -// --force and --prompt are deliberately absent: the manifest path reads neither, but -// neither leaves a false belief behind. There is no uncommitted-changes gate on -// this path for --force to open, and a manifest deploy always asks, which is what -// --prompt was for. Both get the outcome the flag asked for. +// Everywhere else the project decides, by utils.Locate's walk. In or below a +// project in the Astro CLI 1.x layout every mode is refused, naming the two +// ways forward: convert it with astro init, or deploy it with Astro CLI 1.x, +// which keeps deploying that layout. --image-name is refused there too: a +// prebuilt image deployed from a 1.x checkout would leave that project's +// DAGs stale with nothing said. Below a pyproject.toml project's root, every +// mode is told to run from the root. // -// --build-secret has LEFT this list, which is the one entry that went the other -// way. It was blocked on tier 3, then on imagebuild passing a -// --secret; both have landed, so the flag is READ now. It still needs a project -// Dockerfile to be mounted into, and deployManifest refuses it without one — a refusal -// about what the project declares rather than about which version it is, gated on -// the flag being given rather than on a resolved value, since -// util.ResolveBuildSecrets also reads BUILD_SECRET_INPUT from the environment. -var manifestDeployIgnores = []struct { - flag string - do string -}{ - {"pytest", "run your tests before deploying: `uv run pytest && astro deploy`"}, - {"parse", "check your DAGs before deploying: `astro local check && astro deploy`"}, - {"dags-path", "deploy from the project directory; a DAGs path other than dags/ is not supported yet"}, - {"dag-bundle-name", "named DAG bundles are not supported here yet"}, - {"test", "run your tests before deploying: `uv run pytest && astro deploy`"}, - {"env", "this deploy runs no tests, so it reads no test env file; run `uv run pytest` yourself"}, - {"save", "this deploy always asks; mark a link `default = true` in [tool.astro.deployments] to move the cursor"}, - {"deployment-name", "name the target with --deployment, which takes a link name or a Deployment id"}, -} - -// refuseFlagsManifestDeployIgnores stops a manifest deploy that was given a flag it would -// silently drop. -func refuseFlagsManifestDeployIgnores(cmd *cobra.Command) error { - for _, ignored := range manifestDeployIgnores { - if cmd.Flags().Changed(ignored.flag) { - return fmt.Errorf("--%s has no effect when deploying a project with a pyproject.toml ([tool.astro]): %s", ignored.flag, ignored.do) - } - } - return nil -} - +// Outside any project, --image-name is the one deploy: the image alone, with +// no DAGs, as there is no project to take them from, and no git commit. A +// Deployment that takes DAG deploys keeps the DAGs it had, and the deploy +// says so. func deploy(cmd *cobra.Command, args []string) error { - // Route by project type. A project with a pyproject.toml ([tool.astro]) - // takes the manifest deploy path; everything else runs the 1.x path below, - // unchanged. project detection lives in internal/project. - if project.HasManifest(config.WorkingPath) { - return deployManifest(cmd, args) - } - - deploymentID = "" - - // Get deploymentId from args, if passed - if len(args) > 0 { - deploymentID = args[0] - } - - if cmd.Flags().Changed("wait-time") && !waitForDeploy { - return errors.New("cannot use --wait-time with --wait=false") + if nonDags { + return deployNonDagsBundle(cmd, args) } - - if deploymentID == "" || forcePrompt || workspaceID == "" { - var err error - workspaceID, err = coalesceWorkspace() - if err != nil { - return errors.Wrap(err, "failed to find a valid workspace") - } + // One walk routes the deploy and words its refusal. + where := utils.Locate(config.WorkingPath) + if where.AtRoot(config.WorkingPath) { + return deployManifest(cmd, args, &where) } - - if dags && image { - return errors.New("cannot use both --dags and --image together. Run 'astro deploy' to update both your image and dags") - } - - if dagBundleName != "" && image { - return errors.New("cannot use --dag-bundle-name with --image; named Dag bundles apply only to deploys that include Dags") + if _, err := cliout.ParseFormat(deployOutput); err != nil { + return err } - - if cmd.Flags().Changed(imageNameFlag) { - for _, f := range []string{"dags", "dags-path", "no-dags-base-dir", "pytest", "parse", "build-secret", "dag-bundle-name"} { - if cmd.Flags().Changed(f) { - return fmt.Errorf("cannot use --%s with --image-name; --image-name implies an image-only deploy", f) - } - } + // The value, not whether the flag was given: an empty --image-name= + // names no image, and would be a build. + if imageName != "" && where.None() { + return deployManifest(cmd, args, nil) } + // Not a usage mistake, so no usage block under the error. + cmd.SilenceUsage = true + return utils.NoDeployableProject(where, utils.Deploy1xRefusedAstro) +} - if nonDags { - return deployNonDagsBundle(cmd, args) +// refuseFlagCombinations stops a deploy given flags that cannot all apply, +// before anything is read or asked, the way 1.x's deploy did: +// +// - --wait-time sets how long --wait waits, so it needs --wait; +// - --no-dags-base-dir lays out the DAGs a deploy ships, so it needs a +// deploy that ships some: not --image, and not an --image-name deploy +// outside a project, which ships the image alone. +// +// inProject is whether the deploy runs at a pyproject.toml project's root. +func refuseFlagCombinations(cmd *cobra.Command, inProject bool) error { + if cmd.Flags().Changed("wait-time") && !waitForDeploy { + return cliout.Usage(errors.New("cannot use --wait-time with --wait=false")) } - - // Save deploymentId in config if specified - if deploymentID != "" && saveDeployConfig { - err := config.CFG.ProjectDeployment.SetProjectString(deploymentID) - if err != nil { - return errors.Wrap(err, "failed to save deployment id in config") + if cmd.Flags().Changed("no-dags-base-dir") { + switch { + case image: + return cliout.Usage(errors.New("cannot use --no-dags-base-dir with --image: an image-only deploy ships no DAGs")) + case imageName != "" && !inProject: + return cliout.Usage(errors.New("cannot use --no-dags-base-dir with --image-name outside a pyproject.toml project: the image ships alone, with no DAGs")) } } + return nil +} - // An error, not a printed note: returning nil here made a deploy that never - // happened exit 0, so CI reported it as a success. - if hasUncommittedChanges("") && !forceDeploy { - // Not a usage mistake, so no usage block under the error. - cmd.SilenceUsage = true - return errUncommittedChanges - } +// nonDagsDeployJSON is what `astro deploy --non-dags --output json` +// publishes: the bundle deployed and where. Its keys are the ones +// `astro dbt deploy`'s result uses for the same facts. +type nonDagsDeployJSON struct { + Deployment string `json:"deployment"` + DeploymentName string `json:"deployment_name"` + Workspace string `json:"workspace"` + DeployID string `json:"deploy_id"` + // BundleType is --non-dags-bundle-type, BundlePath the directory bundled. + BundleType string `json:"bundle_type"` + BundlePath string `json:"bundle_path"` + MountPath string `json:"mount_path"` + BundleVersion string `json:"bundle_version"` + // Waited is true when the run waited for the Deployment to become + // healthy (--wait), and WaitError says why that wait failed. + Waited bool `json:"waited"` + WaitError string `json:"wait_error,omitempty"` + Git *deployGitJSON `json:"git,omitempty"` +} - // case for astro deploy --dags whose default operation should be not running any tests - if dags && !parse && !pytest { - pytestFile = "" - } else { - pytestFile = deployTests(parse, pytest, forceDeploy, pytestFile) +func newNonDagsDeployJSON(res *astrodeploy.BundleDeploy, bundleType, bundlePath string, waited bool, waitErr error) nonDagsDeployJSON { + return nonDagsDeployJSON{ + Deployment: res.DeploymentID, + DeploymentName: res.DeploymentName, + Workspace: res.WorkspaceID, + DeployID: res.DeployID, + BundleType: bundleType, + BundlePath: bundlePath, + MountPath: res.MountPath, + BundleVersion: res.BundleVersion, + Waited: waited, + WaitError: errText(waitErr), + Git: bundleGitJSON(res.Git), } - - // Silence Usage as we have now validated command input - cmd.SilenceUsage = true - - deployInput := astrodeploy.InputDeploy{ - Path: config.WorkingPath, - RuntimeID: deploymentID, - WsID: workspaceID, - WsIDFromFlag: cmd.Flags().Changed("workspace-id"), - Pytest: pytestFile, - EnvFile: envFile, - ImageName: imageName, - DeploymentName: deploymentName, - Prompt: forcePrompt, - Dags: dags, - NoDagsBaseDir: noDagsBaseDir, - Image: image, - WaitForStatus: waitForDeploy, - WaitTime: waitTime, - Progress: cmd.ErrOrStderr(), - DagsPath: dagsPath, - Description: deployDescription, - BuildSecrets: util.ResolveBuildSecrets(buildSecrets, os.Getenv(util.BuildSecretInputEnv)), - Force: forceDeploy, - DagBundleName: dagBundleName, - } - - return DeployImage(deployInput, astroV1Client, astroV1Alpha1Client) } func deployNonDagsBundle(cmd *cobra.Command, args []string) error { - for _, f := range []string{"dags", "image", imageNameFlag, "dag-bundle-name", "pytest", "parse", "build-secret", "dags-path", "no-dags-base-dir"} { + // Read first, so a bad --output is the usage error reported. + format, err := cliout.ParseFormat(deployOutput) + if err != nil { + return err + } + // The result goes where stdout is now. What the platform code prints on + // the way (the git note, the archive's) is the output text mode has always + // shown; under json it goes to stderr, as astro dbt deploy's does. + out := cmd.OutOrStdout() + defer strayStdoutToStderr(format)() + + for _, f := range []string{"dags", "image", imageNameFlag, "build-secret", "no-dags-base-dir"} { if cmd.Flags().Changed(f) { - return fmt.Errorf("cannot use --%s with --non-dags; --non-dags performs a non-Dag bundle deploy", f) + return cliout.Usage(fmt.Errorf("cannot use --%s with --non-dags; --non-dags performs a non-Dag bundle deploy", f)) } } + if cmd.Flags().Changed("wait-time") && !waitForDeploy { + return cliout.Usage(errors.New("cannot use --wait-time with --wait=false")) + } + if nonDagsMountPath == "" { - return errors.New("--non-dags-mount-path is required with --non-dags") + return cliout.Usage(errors.New("--non-dags-mount-path is required with --non-dags")) } if nonDagsBundlePath == "" { @@ -356,18 +268,13 @@ func deployNonDagsBundle(cmd *cobra.Command, args []string) error { return fmt.Errorf("bundle path %s is not a directory", nonDagsBundlePath) } - withinAstroProject, err := config.IsWithinProjectDir(nonDagsBundlePath) - if err != nil { - return fmt.Errorf("failed to verify bundle path is not within an Astro project: %w", err) - } - if !withinAstroProject { - withinAstroProject = isWithinManifestProject(nonDagsBundlePath) - } - if withinAstroProject { + // The walk every deploy decides by: a pyproject.toml project or a 1.x + // one at or above the bundle path. + if !utils.Locate(nonDagsBundlePath).None() { return errors.New("bundle path is within an Astro project. Non-Dag bundles must be a separate directory") } - targetID, target, err := resolveBundleDeployment(args, workspaceID, deploymentName, "") + targetID, target, login, err := nonDagsTarget(cmd.Context(), args, format) if err != nil { return err } @@ -381,24 +288,126 @@ func deployNonDagsBundle(cmd *cobra.Command, args []string) error { Deployment: target, BundleType: nonDagsBundleType, Description: deployDescription, - AstroV1Client: astroV1Client, + AstroV1Client: login.client, + } + if !login.current { + deployBundleInput.Login = &login.context } res, err := DeployBundle(deployBundleInput) if err != nil { return err } - // A 1.x project's deploy has no --output, so this is text: the line it has - // always printed, then the wait. - return publishThenWait(cmd, cliout.FormatText, waitForDeploy, res.DeploymentID, waitTime, func(error) error { - return renderBundleUploaded(res.BundleVersion)(cmd.OutOrStdout()) + waitDone := func() error { + if login.current { + return waitForBundle(cmd.ErrOrStderr(), res.DeploymentID, waitTime, login.client) + } + return astrodeploy.WaitForBundleIn(cmd.ErrOrStderr(), &login.context, res.DeploymentID, waitTime, login.client) + } + return publishThenWaitWith(cmd, format, waitForDeploy, waitDone, func(waitErr error) error { + if format == cliout.FormatJSON { + return cliout.Renderer{Format: format, Out: out}.Emit(newNonDagsDeployJSON(&res, nonDagsBundleType, nonDagsBundlePath, waitForDeploy, waitErr), nil) + } + return renderBundleUploaded(res.BundleVersion)(out) }) } +// nonDagsTarget is the Deployment a non-Dag bundle deploys to, and the login +// it deploys under, the way astro deploy from the same directory picks them. +// +// The login: in a pyproject.toml project here whose manifest loads, the +// project's (its Astro host's), for everything; anywhere else, or when the +// manifest does not load, the current context's. +// +// The target: the argument or --deployment, which must agree, a link name of +// the project here or a Deployment id. With neither, the workspace's +// Deployments are offered (--workspace, else --workspace-id, else the +// login's): only to a run that can be asked, and only on the current +// context's host, as astro deploy offers them. A run that cannot be asked, +// under --output json or without a terminal, is refused as astro deploy's is, +// with kind input_required. +func nonDagsTarget(ctx context.Context, args []string, format cliout.Format) (string, *astrov1.Deployment, deployLogin, error) { + linkName := "" + if len(args) > 0 { + linkName = args[0] + } + ws := cmp.Or(manifestWorkspace, workspaceID) + // Agreement first, before anything is read. + if _, _, err := manifestdeploy.ResolveNamed(nil, linkName, manifestDeployment, ws, ""); err != nil { + current, _ := loginForDeploy(ctx, "") //nolint:errcheck // with no domain it is the current context, and never fails + return "", nil, current, cliout.Usage(err) + } + + m := manifestHere() + domain := "" + if m != nil { + domain = m.Astro.LoginDomain() + } + login, err := loginForDeploy(ctx, domain) + if err != nil { + return "", nil, login, err + } + + if name := cmp.Or(linkName, manifestDeployment); name != "" { + if m == nil || !hasLink(m, name) { + return name, nil, login, nil + } + named, _, err := manifestdeploy.ResolveNamed(m, name, "", ws, "") + if err != nil { + return "", nil, login, cliout.Usage(err) + } + return named.DeploymentID, nil, login, nil + } + + if format != cliout.FormatText || !stdinIsTerminal() { + return "", nil, login, input.Required(errNonDagsNoTarget) + } + if !login.current { + return "", nil, login, errPickerOtherHost(login.context.Domain) + } + if ws == "" { + ws, err = coalesceWorkspace() + if err != nil { + return "", nil, login, errors.Wrap(err, "failed to find a valid workspace") + } + } + id, read, err := resolveBundleDeployment(nil, ws, "", noCreateUnderJSON(format, "to deploy to")) + return id, read, login, err +} + +// errNonDagsNoTarget is a --non-dags deploy that names no Deployment and +// cannot be asked for one, in astro deploy's words for the same refusal. +var errNonDagsNoTarget = errors.New("this deploy names no Deployment and this run cannot be asked: pass the Deployment id as the argument or with --deployment") + +// errPickerOtherHost is the workspace picker refused on a login other than +// the current context's: it lists only the current context's Deployments. +func errPickerOtherHost(domain string) error { + return fmt.Errorf("this project deploys to %[1]s, and the deployment picker lists only the current context's Deployments. Pass --deployment , or run astro context switch %[1]s", domain) +} + +// manifestHere is the manifest of the pyproject.toml project the working +// directory is the root of, as the walk loaded it, or nil when there is none +// or it did not load. +func manifestHere() *manifest.Manifest { + return utils.Locate(config.WorkingPath).Loaded(config.WorkingPath) +} + +func hasLink(m *manifest.Manifest, name string) bool { + _, ok := m.Astro.Deployments[name] + return ok +} + // deployManifest runs the manifest deploy path: load the manifest, gather flags and // context, resolve the deployment, and run the deploy — dags-only, image-only, // or both — then render the result. The manifest deploy's logic lives in internal/deploy; this // is the cmd shim that parses, wires the transport, and prints. -func deployManifest(cmd *cobra.Command, args []string) error { +// +// root is the walk's answer for the project the deploy runs at the root of, +// carrying its loaded manifest. It is nil for an --image-name deploy outside a +// pyproject.toml project: there is no manifest, so the target is a Deployment id (the +// argument or --deployment) or the workspace's pick, the login is the current +// context's, and the image ships alone, with no DAGs. +func deployManifest(cmd *cobra.Command, args []string, root *utils.Where) error { + inProject := root != nil // The format is read before the flag refusals, so a bad --output is the // usage error reported rather than whichever refusal came first. Every // failure below, refusals included, reaches a json-mode caller as the one @@ -409,16 +418,20 @@ func deployManifest(cmd *cobra.Command, args []string) error { if err != nil { return err } - cmd.SilenceUsage = true - if rerr := refuseFlagsManifestDeployIgnores(cmd); rerr != nil { - return manifestDeployErr(cmd, rerr) + if err := refuseFlagCombinations(cmd, inProject); err != nil { + return err } + cmd.SilenceUsage = true out := cmd.OutOrStdout() - m, err := manifest.Load(filepath.Join(config.WorkingPath, "pyproject.toml")) - if err != nil { - return manifestDeployErr(cmd, err) + // The manifest the walk loaded, or why it did not: it is not read again. + var m *manifest.Manifest + if inProject { + if root.LoadErr != nil { + return manifestDeployErr(cmd, root.LoadErr) + } + m = root.Manifest } // --build-secret is refused HERE, not in internal/deploy, and gated on the @@ -433,22 +446,11 @@ func deployManifest(cmd *cobra.Command, args []string) error { // - --dags never reaches runImage, so a refusal there let // `--dags --build-secret` succeed with the flag silently dropped. // - --image-name returns before any build, so the same silent drop applied - // whenever the project happened to declare a Dockerfile. The 1.x body has a - // cross-flag guard listing build-secret, but a manifest deploy branches away - // before reaching it. + // whenever the project happened to declare a Dockerfile. // // One place, before anything is resolved or prompted for. - if cmd.Flags().Changed("build-secret") { - switch { - case dags: - return manifestDeployErr(cmd, errors.New("--build-secret has no effect with --dags: a dags-only deploy builds no image")) - case imageName != "": - return manifestDeployErr(cmd, errors.New("--build-secret has no effect with --image-name: the image is already built")) - case m.Astro.Dockerfile == "": - if err := util.CheckGeneratedBuildSecrets(buildSecrets); err != nil { - return manifestDeployErr(cmd, err) - } - } + if err := refuseBuildSecret(cmd, m); err != nil { + return manifestDeployErr(cmd, err) } linkName := "" @@ -456,7 +458,13 @@ func deployManifest(cmd *cobra.Command, args []string) error { linkName = args[0] } - login, err := loginForDeploy(cmd.Context(), m.Astro.LoginDomain()) + // Without a project there is no host of its own, and no build to + // declare secrets for. + domain, secretSpecs := "", []string(nil) + if m != nil { + domain, secretSpecs = m.Astro.LoginDomain(), m.Astro.BuildSecretSpecs() + } + login, err := loginForDeploy(cmd.Context(), domain) if err != nil { return manifestDeployErr(cmd, err) } @@ -475,7 +483,7 @@ func deployManifest(cmd *cobra.Command, args []string) error { // Only a run that is going to ask has any use for the ambient layers, and // reading the pin is not free — it creates the project's state directory. - willPrompt := interactive && linkName == "" && manifestDeployment == "" + willPrompt := interactive && inProject && linkName == "" && manifestDeployment == "" preselect, preselectFrom := deployPreselect(config.WorkingPath, willPrompt) errOut := cmd.ErrOrStderr() @@ -489,18 +497,20 @@ func deployManifest(cmd *cobra.Command, args []string) error { WorkspaceID: overrideWorkspace, ContextWorkspace: login.context.Workspace, DagsOnly: dags, - Image: image, - ImageName: imageName, - Description: deployDescription, - Wait: waitForDeploy, - WaitTime: waitTime, - NoDagsBaseDir: noDagsBaseDir, - Interactive: interactive, - // The 1.x path's resolution, so a CI job setting BUILD_SECRET_INPUT + // Outside a project the image ships alone: there are no DAGs here. + Image: image, + ImageAlone: !inProject, + ImageName: imageName, + Description: deployDescription, + Wait: waitForDeploy, + WaitTime: waitTime, + NoDagsBaseDir: noDagsBaseDir, + Interactive: interactive, + // Astro CLI 1.x's resolution, so a CI job setting BUILD_SECRET_INPUT // keeps working across the version boundary rather than silently // losing its secrets on the day the project converts. With neither the // flag nor the variable, the manifest's build-secrets apply. - BuildSecrets: util.ResolveProjectBuildSecrets(buildSecrets, m.Astro.BuildSecretSpecs()), + BuildSecrets: util.ResolveProjectBuildSecrets(buildSecrets, secretSpecs), // The runtime build is checked against the catalog where the image is // about to be built from it. Its warnings go to stderr, so a json run's // stdout stays the one result object. @@ -541,7 +551,19 @@ func deployManifest(cmd *cobra.Command, args []string) error { if res.Git.Uncommitted { fmt.Fprintln(errOut, "note: the project has uncommitted changes, so this deploy records no git commit") } - return renderManifestDeploy(out, format, &res) + var warnings []string + if !inProject { + // The image shipped alone. A Deployment that takes DAG deploys + // keeps the DAGs it had; one that does not runs whatever DAGs the + // image carries, which may be none. + w := fmt.Sprintf(warningDagsInImage, res.DeploymentID, imageName) + if res.DagDeployEnabled { + w = fmt.Sprintf(warningDagsNotUpdated, res.DeploymentID, res.DeploymentID) + } + warnings = append(warnings, w) + fmt.Fprintf(errOut, "warning: %s\n", w) + } + return renderManifestDeploy(out, format, &res, warnings) } // announceDeployTarget prints the one line every resolving command puts on @@ -605,8 +627,25 @@ type deployJSON struct { RuntimeVersion string `json:"runtime_version,omitempty"` URL string `json:"url,omitempty"` Git *deployGitJSON `json:"git,omitempty"` + // Warnings are what the deploy warned about without failing, as text + // prints them on stderr (less the "warning: " prefix): an image shipped + // alone to a Deployment that takes DAG deploys, whose DAGs it left as + // they were. + Warnings []string `json:"warnings,omitempty"` } +// warningDagsNotUpdated is the warning for an --image-name deploy outside a +// project to a Deployment that takes DAG deploys: the image changed, its DAGs +// did not. +const warningDagsNotUpdated = "this deploy shipped the image alone, so Deployment %s keeps the DAGs it had. " + + "To update them, run astro deploy %s --dags from the project directory" + +// warningDagsInImage is the same deploy to a Deployment that takes no DAG +// deploys: it runs the DAGs inside the image, which no project here put +// there. APC's deploy warns the same (warningImageNameDagsInImage). +const warningDagsInImage = "Deployment %s runs the DAGs inside the image %s; no dags folder was deployed. " + + "An image astro package built without a dockerfile declared under [tool.astro] contains none" + type deployGitJSON struct { CommitSHA string `json:"commit_sha"` Branch string `json:"branch,omitempty"` @@ -621,8 +660,9 @@ type deployGitJSON struct { // stream record before it (build progress and warnings go to stderr, and json // mode drops the announce and progress lines), so it is laid out the way // every result is, pretty on a terminal and one line when piped. -func renderManifestDeploy(w io.Writer, format cliout.Format, res *manifestdeploy.Result) error { +func renderManifestDeploy(w io.Writer, format cliout.Format, res *manifestdeploy.Result, warnings []string) error { obj := deployJSON{ + Warnings: warnings, Deployment: res.DeploymentID, Link: res.LinkName, Workspace: res.WorkspaceID, @@ -680,8 +720,8 @@ func manifestDeployErr(cmd *cobra.Command, err error) error { type deployLogin struct { context config.Context client astrov1.APIClient - // current reports the login is the current context's, which the 1.x path's - // deployment picker assumes. + // current reports the login is the current context's, which the + // workspace-level deployment picker assumes. current bool } @@ -726,8 +766,8 @@ var newManifestDeployer = func(login *deployLogin, in io.Reader, errOut io.Write return manifestDeployer{login: login, in: in, errOut: errOut} } -// manifestDeployer wires internal/deploy's transport seam to the 1.x path's cloud/deploy -// transport and the deployment selection flow. +// manifestDeployer wires internal/deploy's transport seam to the Astro deploy +// transport (internal/platform/astro/deploy) and the deployment selection flow. type manifestDeployer struct { login *deployLogin // in and errOut carry the deploy prompt. It asks on stderr and reads stdin, @@ -787,11 +827,11 @@ func (d manifestDeployer) ConfirmTarget(choices []manifestdeploy.Choice, presele return choices[i].Name, nil } -// ResolveUnlinked runs the 1.x path's workspace-level pick/create flow and returns the +// ResolveUnlinked runs the workspace-level pick/create flow and returns the // chosen deployment id. func (d manifestDeployer) ResolveUnlinked(workspaceID string) (string, error) { if !d.login.current { - return "", fmt.Errorf("this project deploys to %[1]s, and the deployment picker lists only the current context's Deployments. Pass --deployment , or run `astro context switch %[1]s`", d.login.context.Domain) + return "", errPickerOtherHost(d.login.context.Domain) } dep, err := deployment.GetDeployment(workspaceID, "", "", false, nil, d.login.client) if err != nil { @@ -800,7 +840,7 @@ func (d manifestDeployer) ResolveUnlinked(workspaceID string) (string, error) { return dep.Id, nil } -// DeployDags reuses the 1.x dags-only transport for the project's dags/. +// DeployDags ships the project's dags/ through the dags-only transport. func (d manifestDeployer) DeployDags(in *manifestdeploy.DagDeploy) (manifestdeploy.DagResult, error) { res, err := astrodeploy.DeployManifestDags(astrodeploy.ManifestDagDeployInput{ Login: &d.login.context, @@ -866,30 +906,25 @@ func (d manifestDeployer) DeployImage(in *manifestdeploy.ImageDeploy) (manifestd DagTarballVersion: res.DagTarballVersion, URL: res.URL, Git: toManifestDeployGit(res.Git), + DagDeployEnabled: res.DagDeployEnabled, }, nil } -// isWithinManifestProject reports whether path sits at or inside a project -// with a pyproject.toml ([tool.astro]). -// -// config.IsWithinProjectDir only knows the 1.x marker, .astro/config.yaml, so it -// answers false at every level of a project with a pyproject.toml. That left both containment -// refusals in this package unreachable for exactly the projects that have a manifest: -// a dbt project or a non-DAG bundle nested inside one shipped where the check -// meant to stop it. -func isWithinManifestProject(path string) bool { - abs, err := filepath.Abs(filepath.Clean(path)) - if err != nil { - return false - } - for dir := abs; ; { - if project.HasManifest(dir) { - return true - } - parent := filepath.Dir(dir) - if parent == dir { - return false - } - dir = parent +// refuseBuildSecret is deployManifest's --build-secret check (see the +// comment where it is called): the flag where nothing is built, or a secret +// a generated build cannot mount. m is nil only for an --image-name deploy +// outside a project, which the second case stops first. +func refuseBuildSecret(cmd *cobra.Command, m *manifest.Manifest) error { + if !cmd.Flags().Changed("build-secret") { + return nil + } + switch { + case dags: + return errors.New("--build-secret has no effect with --dags: a dags-only deploy builds no image") + case imageName != "": + return errors.New("--build-secret has no effect with --image-name: the image is already built") + case m.Astro.Dockerfile == "": + return util.CheckGeneratedBuildSecrets(buildSecrets) } + return nil } diff --git a/cmd/astro/deploy_help.go b/cmd/astro/deploy_help.go index d5dff7657..8ebbdfaec 100644 --- a/cmd/astro/deploy_help.go +++ b/cmd/astro/deploy_help.go @@ -12,7 +12,6 @@ import ( const ( deployGroupImage = "Image" deployGroupDAG = "Dag" - deployGroupTest = "Test" deployGroupNonDAG = "Non-Dag Bundle" ) @@ -30,12 +29,12 @@ func annotateDeployFlag(cmd *cobra.Command, name, group string) { } // orderDeployFlagGroups lists deploy's sections in the order a deploy narrows: -// what image ships, which DAGs, what must pass first, and what rides beside. +// what image ships, which DAGs, and what rides beside. func orderDeployFlagGroups(cmd *cobra.Command) { if cmd.Annotations == nil { cmd.Annotations = map[string]string{} } cmd.Annotations[flagGroupOrderAnnotation] = strings.Join([]string{ - deployGroupImage, deployGroupDAG, deployGroupTest, deployGroupNonDAG, + deployGroupImage, deployGroupDAG, deployGroupNonDAG, }, ",") } diff --git a/cmd/astro/deploy_manifest_test.go b/cmd/astro/deploy_manifest_test.go index 6bb12e07f..b29ec4004 100644 --- a/cmd/astro/deploy_manifest_test.go +++ b/cmd/astro/deploy_manifest_test.go @@ -20,7 +20,6 @@ import ( "github.com/astronomer/astro-cli/internal/astrosession" manifestdeploy "github.com/astronomer/astro-cli/internal/deploy" astrov1 "github.com/astronomer/astro-cli/internal/platform/astro/clients/astrov1" - astrov1alpha1 "github.com/astronomer/astro-cli/internal/platform/astro/clients/astrov1alpha1" astrodeploy "github.com/astronomer/astro-cli/internal/platform/astro/deploy" "github.com/astronomer/astro-cli/pkg/input" "github.com/astronomer/astro-cli/pkg/instances" @@ -49,8 +48,6 @@ func resetDeployFlagVars() { manifestWorkspace = "" noDagsBaseDir = false waitForDeploy = false - forceDeploy = false - forcePrompt = false workspaceID = "" deploymentName = "" deployDescription = "" @@ -773,86 +770,13 @@ func TestDeployRoutesManifestProject(t *testing.T) { assert.Contains(t, err.Error(), "--deployment") } -func TestDeployRoutes1xProject(t *testing.T) { - testUtil.InitTestConfig(testUtil.LocalPlatform) - resetDeployFlagVars() - - dir := t.TempDir() // no pyproject.toml, so not a project - - orig := config.WorkingPath - config.WorkingPath = dir - t.Cleanup(func() { config.WorkingPath = orig }) - - origEnsure := EnsureProjectDir - origDeploy := DeployImage - t.Cleanup(func() { - EnsureProjectDir = origEnsure - DeployImage = origDeploy - }) - - EnsureProjectDir = func(cmd *cobra.Command, args []string) error { return nil } - called := false - DeployImage = func(astrodeploy.InputDeploy, astrov1.APIClient, astrov1alpha1.APIClient) error { - called = true - return nil - } - - err := execDeployCmd("test-deployment-id", "-f", "--workspace-id", "test-ws") - require.NoError(t, err) - assert.True(t, called, "a 1.x project should run the 1.x deploy path") -} - -// Eight flags reach astro deploy that the manifest path never reads. Accepting them -// silently means a deploy someone believes ran their tests, shipped from a -// DAGs path it never looked at, or saved a target it did not save. Each is -// refused with what to do instead until the ones worth porting are ported. -// -// Eight rather than nine: --build-secret is deliberately NOT in this table any -// more, because the manifest path READS it now. It still needs a project Dockerfile -// to be mounted into, and that refusal lives with the other build-secret -// checks in TestDeployManifestBuildSecretRefusals — gated on the flag being given, -// which is why it cannot be a row here. -func TestDeployRefusesFlagsTheManifestPathIgnores(t *testing.T) { - cases := []struct { - args []string - want string - }{ - {[]string{"--pytest"}, "uv run pytest"}, - {[]string{"--parse"}, "astro local check"}, - {[]string{"--dags-path", "./elsewhere"}, "not supported yet"}, - {[]string{"--dag-bundle-name", "nightly"}, "not supported here yet"}, - {[]string{"--test", "tests/test_dags.py"}, "uv run pytest"}, - {[]string{"--env", ".env.ci"}, "runs no tests"}, - {[]string{"--save"}, "always asks"}, - {[]string{"--deployment-name", "prod"}, "--deployment"}, - } - - for _, tc := range cases { - t.Run(tc.args[0], func(t *testing.T) { - testUtil.InitTestConfig(testUtil.LocalPlatform) - resetDeployFlagVars() - - dir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dir, "pyproject.toml"), []byte(manifestForRouting), 0o600)) - orig := config.WorkingPath - config.WorkingPath = dir - t.Cleanup(func() { config.WorkingPath = orig }) - - out, err := execDeployCapture(append([]string{actionDeploymentID}, tc.args...)...) - require.Error(t, err) - assert.Contains(t, err.Error(), "has no effect when deploying a project with a pyproject.toml") - assert.Contains(t, err.Error(), tc.want) - assert.NotContains(t, out, "Usage:", "a refusal prints the error, not the help") - }) - } -} - -// --force and --prompt are read by neither path but refused by neither either. -// The manifest path has no uncommitted-changes gate for --force to open, and it always -// asks, which is what --prompt requested — so both get the outcome the flag -// asked for and refusing them would break CI that passes them out of habit. +// --force and --prompt are accepted and read by nothing: there is no +// uncommitted-changes gate for --force to open (and astronomer/deploy-action +// passes it on every deploy), and the deploy always asks, which is what +// --prompt requested. The run gets as far as asking for --deployment, as one +// without them does. func TestDeployAcceptsForceAndPromptOnAManifestProject(t *testing.T) { - for _, flag := range []string{"--force", "--prompt"} { + for _, flag := range []string{"--force", "-f", "--prompt", "-p"} { t.Run(flag, func(t *testing.T) { testUtil.InitTestConfig(testUtil.LocalPlatform) resetDeployFlagVars() @@ -864,30 +788,12 @@ func TestDeployAcceptsForceAndPromptOnAManifestProject(t *testing.T) { t.Cleanup(func() { config.WorkingPath = orig }) err := execDeployCmd(flag) - if err != nil { - assert.NotContains(t, err.Error(), "has no effect when deploying a project with a pyproject.toml") - } + require.Error(t, err) + assert.Contains(t, err.Error(), "--deployment") }) } } -// The same flags still work on a 1.x project, where the 1.x deploy path reads -// them. The refusal is on the manifest branch only. -func TestDeployAllowsThoseFlagsOnA1xProject(t *testing.T) { - testUtil.InitTestConfig(testUtil.LocalPlatform) - resetDeployFlagVars() - - dir := t.TempDir() // no pyproject.toml, so the 1.x path - orig := config.WorkingPath - config.WorkingPath = dir - t.Cleanup(func() { config.WorkingPath = orig }) - - err := execDeployCmd("--pytest") - if err != nil { - assert.NotContains(t, err.Error(), "has no effect when deploying a project with a pyproject.toml") - } -} - // --build-secret is refused where it cannot work, and only when the USER asked. // // Three regressions are pinned here, all from validating in internal/deploy @@ -1091,7 +997,10 @@ func setupDeployOnStage(t *testing.T, toml string) *deployLogin { return picked } -func logInTo(t *testing.T, domain, token, org string) { +// logInToProd stores a login for astronomer.io, the host manifestOnProd names. +func logInToProd(t *testing.T) { + const token, org = "Bearer prod-token", "prod-org" + const domain = "astronomer.io" t.Helper() login := config.Context{Domain: domain} require.NoError(t, login.SetContextKey("token", token)) @@ -1105,7 +1014,7 @@ func TestDeployManifestUsesTheManifestDomainsLogin(t *testing.T) { for _, args := range [][]string{{"test", "--dags"}, {"--deployment", "clx-bare", "--dags"}} { t.Run(strings.Join(args, " "), func(t *testing.T) { picked := setupDeployOnStage(t, manifestOnProd) - logInTo(t, "astronomer.io", "Bearer prod-token", "prod-org") + logInToProd(t) _, err := execDeployCapture(args...) require.NoError(t, err) @@ -1119,11 +1028,102 @@ func TestDeployManifestUsesTheManifestDomainsLogin(t *testing.T) { } } +// --non-dags from a project deploys under the project's host as the project's +// deploy does, a link and a bare Deployment id alike; outside a project a +// bare id goes under the current context. +func TestDeployNonDagsUsesTheManifestDomainsLogin(t *testing.T) { + for _, args := range [][]string{{"test"}, {"--deployment", "clx-bare"}} { + t.Run(strings.Join(args, " "), func(t *testing.T) { + setupDeployOnStage(t, manifestOnProd) + logInToProd(t) + var captured *astrodeploy.DeployBundleInput + prev := DeployBundle + t.Cleanup(func() { DeployBundle = prev }) + DeployBundle = func(in *astrodeploy.DeployBundleInput) (astrodeploy.BundleDeploy, error) { + captured = in + return astrodeploy.BundleDeploy{}, nil + } + + _, err := execDeployCapture(append(args, "--non-dags", "--non-dags-mount-path", "/x", "--non-dags-local-path", t.TempDir())...) + require.NoError(t, err) + require.NotNil(t, captured.Login, "deployed under the project's host") + assert.Equal(t, "astronomer.io", captured.Login.Domain) + assert.Equal(t, "prod-org", captured.Login.Organization) + }) + } + + t.Run("outside a project", func(t *testing.T) { + setupDeployOnStage(t, manifestOnProd) + require.NoError(t, os.Remove(filepath.Join(config.WorkingPath, "pyproject.toml"))) + var captured *astrodeploy.DeployBundleInput + prev := DeployBundle + t.Cleanup(func() { DeployBundle = prev }) + DeployBundle = func(in *astrodeploy.DeployBundleInput) (astrodeploy.BundleDeploy, error) { + captured = in + return astrodeploy.BundleDeploy{}, nil + } + _, err := execDeployCapture("--deployment", "clx-bare", "--non-dags", "--non-dags-mount-path", "/x", "--non-dags-local-path", t.TempDir()) + require.NoError(t, err) + assert.Nil(t, captured.Login, "the current context") + }) +} + +// --non-dags naming no Deployment: a run that cannot be asked is refused as +// astro deploy's is, input_required, before any picker; one that can is +// offered the picker only on the current context's host, so from a project +// on another host it is refused with the same words astro deploy uses. +func TestDeployNonDagsWithNoTarget(t *testing.T) { + noBundle := func(t *testing.T) { + prev := DeployBundle + t.Cleanup(func() { DeployBundle = prev }) + DeployBundle = func(*astrodeploy.DeployBundleInput) (astrodeploy.BundleDeploy, error) { + t.Fatal("no target, so nothing is deployed") + return astrodeploy.BundleDeploy{}, nil + } + } + nonDags := []string{"--non-dags", "--non-dags-mount-path", "/x"} + + t.Run("under --output json", func(t *testing.T) { + setupDeployOnStage(t, manifestOnProd) + logInToProd(t) + noBundle(t) + out, err := execDeployCapture(append(nonDags, "--non-dags-local-path", t.TempDir(), "--output", "json")...) + require.Error(t, err) + var obj struct { + Error string `json:"error"` + Kind string `json:"kind"` + } + require.NoError(t, json.Unmarshal([]byte(out), &obj), out) + assert.Equal(t, "input_required", obj.Kind) + assert.Contains(t, obj.Error, "pass the Deployment id as the argument or with --deployment") + }) + + t.Run("without a terminal", func(t *testing.T) { + setupDeployOnStage(t, manifestOnProd) + logInToProd(t) + noBundle(t) + _, err := execDeployCapture(append(nonDags, "--non-dags-local-path", t.TempDir())...) + require.Error(t, err) + assert.True(t, input.IsRequired(err), "%v", err) + }) + + t.Run("the picker on another host", func(t *testing.T) { + setupDeployOnStage(t, manifestOnProd) + logInToProd(t) + interactiveDeploy(t) + noBundle(t) + _, err := execDeployCapture(append(nonDags, "--non-dags-local-path", t.TempDir())...) + require.Error(t, err) + assert.Contains(t, err.Error(), "the deployment picker lists only the current context's Deployments") + assert.Contains(t, err.Error(), "astro context switch astronomer.io") + }) +} + // ASTRO_API_TOKEN outranks the stored login, and goes out with its scheme like // a stored token, since the deploy's CI/CD check splits it off. func TestDeployManifestSendsTheAPITokenToTheManifestDomain(t *testing.T) { picked := setupDeployOnStage(t, manifestOnProd) - logInTo(t, "astronomer.io", "Bearer prod-token", "prod-org") + logInToProd(t) t.Setenv(astrosession.EnvAPIToken, "ci-token") _, err := execDeployCapture("test", "--dags") diff --git a/cmd/astro/deploy_test.go b/cmd/astro/deploy_test.go index 1e53178d5..812d00b39 100644 --- a/cmd/astro/deploy_test.go +++ b/cmd/astro/deploy_test.go @@ -1,22 +1,26 @@ package astro import ( - "bytes" + "encoding/json" + "errors" + "fmt" + "io" "os" "path/filepath" "testing" - "github.com/spf13/afero" - "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "github.com/astronomer/astro-cli/cmd/cliout" + "github.com/astronomer/astro-cli/cmd/utils" "github.com/astronomer/astro-cli/config" + manifestdeploy "github.com/astronomer/astro-cli/internal/deploy" "github.com/astronomer/astro-cli/internal/platform/astro/clients/astrov1" astrov1_mocks "github.com/astronomer/astro-cli/internal/platform/astro/clients/astrov1/mocks" - astrov1alpha1 "github.com/astronomer/astro-cli/internal/platform/astro/clients/astrov1alpha1" astrodeploy "github.com/astronomer/astro-cli/internal/platform/astro/deploy" + "github.com/astronomer/astro-cli/internal/project" testUtil "github.com/astronomer/astro-cli/pkg/testing" ) @@ -32,237 +36,212 @@ func execDeployCmd(args ...string) error { return err } -// A 1.x project with uncommitted changes is refused with an error, so the -// deploy that did not happen exits non-zero. It used to print a note and -// return nil: exit 0, and CI reported a deploy that never ran. --force deploys -// anyway. -func TestDeployRefusesUncommittedChanges(t *testing.T) { - testUtil.InitTestConfig(testUtil.LocalPlatform) - EnsureProjectDir = func(cmd *cobra.Command, args []string) error { return nil } - prev := hasUncommittedChanges - hasUncommittedChanges = func(string) bool { return true } - t.Cleanup(func() { hasUncommittedChanges = prev }) - - deployed := 0 - DeployImage = func(astrodeploy.InputDeploy, astrov1.APIClient, astrov1alpha1.APIClient) error { - deployed++ - return nil - } - - testUtil.SetupOSArgsForGinkgo() - cmd := NewDeployCmd() - var printed bytes.Buffer - cmd.SetOut(&printed) - cmd.SetErr(&printed) - cmd.SetArgs([]string{"test-deployment-id"}) - _, err := cmd.ExecuteC() - assert.ErrorIs(t, err, errUncommittedChanges) - assert.Equal(t, 0, deployed, "a refused deploy ships nothing") - assert.NotContains(t, printed.String(), "Usage:", "the refusal is not a usage mistake") - - assert.NoError(t, execDeployCmd("test-deployment-id", "--force")) - assert.Equal(t, 1, deployed, "--force deploys anyway") +// deployIn points the deploy at dir for one test. +func deployIn(t *testing.T, dir string) { + t.Helper() + prev := config.WorkingPath + config.WorkingPath = dir + t.Cleanup(func() { config.WorkingPath = prev }) } -func TestDeployImage(t *testing.T) { - testUtil.InitTestConfig(testUtil.LocalPlatform) - - EnsureProjectDir = func(cmd *cobra.Command, args []string) error { - return nil - } +// make1xProject lays out a project the way Astro CLI 1.x made one. +func make1xProject(t *testing.T) string { + t.Helper() + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "Dockerfile"), []byte("FROM quay.io/astronomer/astro-runtime:12.0.0\n"), 0o600)) + require.NoError(t, os.MkdirAll(filepath.Join(dir, ".astro"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, ".astro", "config.yaml"), []byte("project:\n name: demo\n"), 0o600)) + require.NoError(t, os.MkdirAll(filepath.Join(dir, "dags"), 0o755)) + return dir +} - DeployImage = func(deployInput astrodeploy.InputDeploy, astroV1Client astrov1.APIClient, astroV1Alpha1Client astrov1alpha1.APIClient) error { +// noTransport fails the test if the deploy reaches the transport. +func noTransport(t *testing.T) { + t.Helper() + prevDeployer := newManifestDeployer + t.Cleanup(func() { newManifestDeployer = prevDeployer }) + newManifestDeployer = func(*deployLogin, io.Reader, io.Writer) manifestdeploy.Deployer { + t.Fatal("a refused deploy reaches no transport") return nil } - - err := execDeployCmd("-f") - assert.NoError(t, err) - - err = execDeployCmd("test-deployment-id", "-f", "--wait") - assert.NoError(t, err) - - err = execDeployCmd("test-deployment-id", "--save") - assert.NoError(t, err) - - err = execDeployCmd("-f", "test-deployment-id", "--pytest") - assert.NoError(t, err) - - err = execDeployCmd("-f", "test-deployment-id", "--parse") - assert.NoError(t, err) - - err = execDeployCmd("-f", "test-deployment-id", "--parse", "--pytest") - assert.NoError(t, err) - - err = execDeployCmd("test-deployment-id", "--parse", "--pytest") - assert.NoError(t, err) - - err = execDeployCmd("test-deployment-id", "--dags") - assert.NoError(t, err) - - err = execDeployCmd("test-deployment-id", "--dags", "--wait") - assert.NoError(t, err) - - err = execDeployCmd("-f", "test-deployment-id", "--dags", "--pytest") - assert.NoError(t, err) - - err = execDeployCmd("-f", "test-deployment-id", "--dags", "--parse") - assert.NoError(t, err) - - err = execDeployCmd("-f", "test-deployment-id", "--dags", "--parse", "--pytest") - assert.NoError(t, err) } -func TestDeployReturnsErrorWhenSaveConfigFails(t *testing.T) { +// v2 deploys only pyproject.toml projects. A project in the Astro CLI 1.x +// layout is refused for every mode that reads the project, naming the two +// ways forward, and the transport is never reached. Outside any project the +// deploy gives the no-project advice. Both are no_project failures. +func TestDeployRefusesA1xProject(t *testing.T) { testUtil.InitTestConfig(testUtil.LocalPlatform) - - EnsureProjectDir = func(cmd *cobra.Command, args []string) error { - return nil + resetDeployFlagVars() + noTransport(t) + + oneX := make1xProject(t) + deployIn(t, oneX) + for _, args := range [][]string{ + {"test-deployment-id"}, + {"test-deployment-id", "--dags"}, + {"test-deployment-id", "--image"}, + {"test-deployment-id", "--image-name", "img:1"}, + {"--deployment", "test-deployment-id", "--image-name", "img:1", "--force"}, + {"test-deployment-id", "--force", "--output", "json"}, + } { + err := execDeployCmd(args...) + require.Error(t, err, "%v", args) + assert.EqualError(t, err, utils.Deploy1xRefusedAstro("this project", ""), "%v", args) + var notFound *project.NotFoundError + assert.True(t, errors.As(err, ¬Found), "%v: reported as no_project", args) } - DeployImage = func(deployInput astrodeploy.InputDeploy, astroV1Client astrov1.APIClient, astroV1Alpha1Client astrov1alpha1.APIClient) error { - return nil + // From below the project, the refusal names it. + deployIn(t, filepath.Join(oneX, "dags")) + err := execDeployCmd("test-deployment-id") + require.Error(t, err) + assert.Contains(t, err.Error(), "inside a project at "+oneX) + + deployIn(t, t.TempDir()) + for _, args := range [][]string{{"test-deployment-id"}, {"test-deployment-id", "--dags"}, {"test-deployment-id", "--image-name="}} { + err := execDeployCmd(args...) + require.Error(t, err, "%v", args) + assert.Contains(t, err.Error(), "this is not an Astro project directory. Change to an Astro project directory, or run astro init", "%v", args) + assert.NotContains(t, err.Error(), "1.x", "%v", args) + var notFound *project.NotFoundError + assert.True(t, errors.As(err, ¬Found), "%v: reported as no_project", args) } - // config.InitConfig reads and writes real files on disk (via afero.NewOsFs()), - // including the home config that stores the current context. Point ASTRO_HOME at - // a throwaway directory with its own valid context, so the test does not depend on - // (or clobber) whatever real ~/.astro/config.yaml happens to exist on the machine - // running the test - on a fresh CI runner there is no such file, so without this - // the deploy fails earlier with "no context set" instead of exercising the save path. - // - // ASTRO_HOME is restored (and config's cached home-config path resynced) before - // testUtil.InitTestConfig runs in the deferred cleanup below, otherwise the fake - // config it writes for later tests lands at the wrong path and leaves them with no - // context set either. - origAstroHome, hadAstroHome := os.LookupEnv("ASTRO_HOME") - homeDir := t.TempDir() - require.NoError(t, os.Setenv("ASTRO_HOME", homeDir)) - homeAstroDir := filepath.Join(homeDir, config.ConfigDir) - require.NoError(t, os.MkdirAll(homeAstroDir, 0o755)) - require.NoError(t, os.WriteFile(filepath.Join(homeAstroDir, config.ConfigFileNameWithExt), testUtil.NewTestConfig(testUtil.LocalPlatform), 0o600)) - - // Point the project config at a real directory, but make the config file itself - // a directory instead of a file, so the write that saves the deployment ID fails. - // This confirms that a save failure stops the deploy instead of returning nil. - tmpDir := t.TempDir() - astroDir := filepath.Join(tmpDir, config.ConfigDir) - require.NoError(t, os.MkdirAll(filepath.Join(astroDir, config.ConfigFileNameWithExt), 0o755)) - - origWorkingPath := config.WorkingPath - config.WorkingPath = tmpDir - config.InitConfig(afero.NewOsFs()) - defer func() { - if hadAstroHome { - os.Setenv("ASTRO_HOME", origAstroHome) - } else { - os.Unsetenv("ASTRO_HOME") - } - config.WorkingPath = origWorkingPath - config.InitConfig(afero.NewOsFs()) - testUtil.InitTestConfig(testUtil.LocalPlatform) - }() - - err := execDeployCmd("test-deployment-id", "--save") - assert.Error(t, err) - assert.Contains(t, err.Error(), "failed to save deployment id in config") + // A bad --output is the usage error, wherever the deploy runs. + _, err = execDeployCapture("test-deployment-id", "--output", "yaml") + assert.True(t, cliout.IsUsage(err), "%v", err) } -func TestDeploySkipsEnsureProjectDirWhenImageNameSet(t *testing.T) { +// imageDeployer swaps in a fake transport for one test. +func imageDeployer(t *testing.T, fake *fakeCmdDeployer) { + t.Helper() testUtil.InitTestConfig(testUtil.LocalPlatform) + resetDeployFlagVars() + prevDeployer := newManifestDeployer + t.Cleanup(func() { newManifestDeployer = prevDeployer }) + newManifestDeployer = func(*deployLogin, io.Reader, io.Writer) manifestdeploy.Deployer { return fake } +} - EnsureProjectDir = func(cmd *cobra.Command, args []string) error { - return assert.AnError - } - defer func() { - EnsureProjectDir = func(cmd *cobra.Command, args []string) error { return nil } - }() - - DeployImage = func(deployInput astrodeploy.InputDeploy, astroV1Client astrov1.APIClient, astroV1Alpha1Client astrov1alpha1.APIClient) error { - return nil +// Outside any project --image-name deploys the image alone, never the working +// directory's dags/, to the Deployment the argument or --deployment names by +// id. A Deployment that takes DAG deploys keeps the DAGs it had, which the +// deploy says on stderr and in the json result's warnings. +func TestDeployImageNameOutsideAProject(t *testing.T) { + for _, dagDeploy := range []bool{false, true} { + t.Run(fmt.Sprintf("dag deploy %v", dagDeploy), func(t *testing.T) { + fake := &fakeCmdDeployer{img: manifestdeploy.ImageResult{ImageTag: "deploy-1", WorkspaceID: "clw-ws", DagDeployEnabled: dagDeploy}} + imageDeployer(t, fake) + deployIn(t, t.TempDir()) + + out, err := execDeployCapture("--deployment", actionDeploymentID, "--image-name", "img:1", "--output", "json") + require.NoError(t, err, out) + require.NotNil(t, fake.imgInput) + assert.Equal(t, actionDeploymentID, fake.imgInput.DeploymentID) + assert.Equal(t, "img:1", fake.imgInput.ImageName) + assert.False(t, fake.imgInput.IncludeDags, "no DAGs ship from outside a project") + assert.Contains(t, out, `"type":"image-only"`) + // With DAG deploys the Deployment keeps its DAGs; without, it + // runs whatever DAGs the image carries. + warning := fmt.Sprintf(warningDagsInImage, actionDeploymentID, "img:1") + if dagDeploy { + warning = fmt.Sprintf(warningDagsNotUpdated, actionDeploymentID, actionDeploymentID) + } + assert.Contains(t, out, `"warnings":["`+warning+`"]`) + assert.Contains(t, out, "warning: "+warning) + }) } - // With --image-name, the project-dir check should be skipped. - err := execDeployCmd("-f", "test-deployment-id", "--image-name", "custom-image:latest") - assert.NoError(t, err) - - // Without --image-name, the project-dir check should still run and propagate. - err = execDeployCmd("-f", "test-deployment-id") - assert.ErrorIs(t, err, assert.AnError) + t.Run("--dags with it names only the flag given", func(t *testing.T) { + fake := &fakeCmdDeployer{} + imageDeployer(t, fake) + deployIn(t, t.TempDir()) + err := execDeployCmd(actionDeploymentID, "--image-name", "img:1", "--dags") + assert.EqualError(t, err, "--dags deploys only your DAGs; drop --image-name") + assert.Nil(t, fake.imgInput) + assert.Nil(t, fake.dagInput) + }) + + t.Run("no target, and none can be asked for", func(t *testing.T) { + fake := &fakeCmdDeployer{} + imageDeployer(t, fake) + deployIn(t, t.TempDir()) + _, err := execDeployCapture("--image-name", "img:1", "--workspace", "clw-ws", "--output", "json") + require.Error(t, err) + assert.Contains(t, err.Error(), "pass the Deployment id as the argument or with --deployment") + assert.NotContains(t, err.Error(), "project") + }) + + t.Run("below a pyproject.toml project's root it is refused", func(t *testing.T) { + imageDeployer(t, &fakeCmdDeployer{}) + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "pyproject.toml"), []byte(manifestForRouting), 0o600)) + require.NoError(t, os.MkdirAll(filepath.Join(dir, "dags"), 0o755)) + deployIn(t, filepath.Join(dir, "dags")) + err := execDeployCmd(actionDeploymentID, "--image-name", "img:1") + assert.ErrorContains(t, err, "Run the deploy from the project directory, "+dir) + }) } -func TestDeploySkipsEnsureProjectDirWhenDagsPathSet(t *testing.T) { - testUtil.InitTestConfig(testUtil.LocalPlatform) - - EnsureProjectDir = func(cmd *cobra.Command, args []string) error { - return assert.AnError - } - defer func() { - EnsureProjectDir = func(cmd *cobra.Command, args []string) error { return nil } - }() - - DeployImage = func(deployInput astrodeploy.InputDeploy, astroV1Client astrov1.APIClient, astroV1Alpha1Client astrov1alpha1.APIClient) error { - return nil +// The flag combinations 1.x refused, where they still apply: --wait-time needs +// --wait, and --no-dags-base-dir needs a deploy that ships DAGs. +func TestDeployRefusesFlagsThatCannotAllApply(t *testing.T) { + for _, tc := range []struct { + name string + inProject bool + args []string + says string + }{ + {"--wait-time without --wait", true, []string{actionDeploymentID, "--wait-time", "1m"}, "cannot use --wait-time with --wait=false"}, + {"--no-dags-base-dir with --image", true, []string{actionDeploymentID, "--image", "--no-dags-base-dir"}, "cannot use --no-dags-base-dir with --image"}, + {"--no-dags-base-dir with --image-name outside a project", false, []string{actionDeploymentID, "--image-name", "img:1", "--no-dags-base-dir"}, "cannot use --no-dags-base-dir with --image-name outside a pyproject.toml project"}, + } { + t.Run(tc.name, func(t *testing.T) { + fake := &fakeCmdDeployer{} + imageDeployer(t, fake) + dir := t.TempDir() + if tc.inProject { + require.NoError(t, os.WriteFile(filepath.Join(dir, "pyproject.toml"), []byte(manifestForRouting), 0o600)) + } + deployIn(t, dir) + _, err := execDeployCapture(tc.args...) + require.Error(t, err) + assert.Contains(t, err.Error(), tc.says) + assert.True(t, cliout.IsUsage(err)) + assert.Nil(t, fake.imgInput) + }) } - // With --dags and --dags-path, the DAGs are read entirely from --dags-path, so the - // project-dir check should be skipped. - err := execDeployCmd("-f", "test-deployment-id", "--dags", "--dags-path", "./external-dags") - assert.NoError(t, err) - - // --dags alone (DAGs read from the working directory) still needs the project-dir check. - err = execDeployCmd("-f", "test-deployment-id", "--dags") - assert.ErrorIs(t, err, assert.AnError) - - // --dags-path alone (without --dags, i.e. a full image+dags deploy) still needs the project - // root as the Docker build context. - err = execDeployCmd("-f", "test-deployment-id", "--dags-path", "./external-dags") - assert.ErrorIs(t, err, assert.AnError) - - // --pytest/--parse build and run a local image to test-parse the DAGs, so the project-dir - // check should still run even with --dags-path set. - err = execDeployCmd("-f", "test-deployment-id", "--dags", "--dags-path", "./external-dags", "--pytest") - assert.ErrorIs(t, err, assert.AnError) - - err = execDeployCmd("-f", "test-deployment-id", "--dags", "--dags-path", "./external-dags", "--parse") - assert.ErrorIs(t, err, assert.AnError) - - // Explicitly passing --dags=false must not be misread as enabling the bypass just because - // the flag was mentioned on the command line - this is really a full image+dags deploy. - err = execDeployCmd("-f", "test-deployment-id", "--dags=false", "--dags-path", "./external-dags") - assert.ErrorIs(t, err, assert.AnError) - - // An explicit empty --dags-path falls back to the working directory's dags/ folder - // (cloud/deploy/deploy.go), so it must not be misread as enabling the bypass either. - err = execDeployCmd("-f", "test-deployment-id", "--dags", "--dags-path", "") - assert.ErrorIs(t, err, assert.AnError) + // In a project --image-name ships the project's DAGs, so their layout + // is the flag's to set. + fake := &fakeCmdDeployer{} + imageDeployer(t, fake) + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "pyproject.toml"), []byte(manifestForRouting), 0o600)) + deployIn(t, dir) + _, err := execDeployCapture(actionDeploymentID, "--image-name", "img:1", "--no-dags-base-dir") + require.NoError(t, err) + require.NotNil(t, fake.imgInput) + assert.True(t, fake.imgInput.NoDagsBaseDir) } -func TestDeployImageNameRejectsIncompatibleFlags(t *testing.T) { - testUtil.InitTestConfig(testUtil.LocalPlatform) - - EnsureProjectDir = func(cmd *cobra.Command, args []string) error { return nil } - DeployImage = func(deployInput astrodeploy.InputDeploy, astroV1Client astrov1.APIClient, astroV1Alpha1Client astrov1alpha1.APIClient) error { - return nil +// The flags only the 1.x deploy read are gone from the command: a run that +// passes one fails as an unknown flag here, and the root's removed-flags +// registry names what replaced it (cmd's TestRemovedFlagsSayWhatReplacedThem). +// --force and --prompt stay, hidden and read by nothing: astronomer/deploy-action +// passes --force, and a pyproject.toml project's deploy always took both. +func TestDeployDropsThe1xOnlyFlags(t *testing.T) { + cmd := NewDeployCmd() + for _, name := range []string{"save", "pytest", "parse", "test", "env", "dags-path", "dag-bundle-name", "deployment-name"} { + assert.Nil(t, cmd.Flags().Lookup(name), "--%s", name) } - - cases := []struct { - name string - args []string - }{ - {"dags", []string{"-f", "test-deployment-id", "--image-name", "img:1", "--dags"}}, - {"dags-path", []string{"-f", "test-deployment-id", "--image-name", "img:1", "--dags-path", "./dags"}}, - {"no-dags-base-dir", []string{"-f", "test-deployment-id", "--image-name", "img:1", "--no-dags-base-dir"}}, - {"pytest", []string{"-f", "test-deployment-id", "--image-name", "img:1", "--pytest"}}, - {"parse", []string{"-f", "test-deployment-id", "--image-name", "img:1", "--parse"}}, - {"build-secret", []string{"-f", "test-deployment-id", "--image-name", "img:1", "--build-secret", "id=mysecret,src=secrets.txt"}}, - {"multiple build-secret", []string{"-f", "test-deployment-id", "--image-name", "img:1", "--build-secret", "id=mysecret,src=secrets.txt", "--build-secret", "id=aws,src=credentials"}}, + for _, letter := range []string{"s", "t", "e", "n"} { + assert.Nil(t, cmd.Flags().ShorthandLookup(letter), "-%s", letter) } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - err := execDeployCmd(tc.args...) - assert.Error(t, err) - assert.Contains(t, err.Error(), "--image-name") - }) + for _, name := range []string{"force", "prompt"} { + f := cmd.Flags().Lookup(name) + require.NotNil(t, f, name) + assert.True(t, f.Hidden, name) } } @@ -355,14 +334,23 @@ func (s *NonDagsDeploySuite) TestProvidedDeploymentId() { assert.Equal(s.T(), s.tmpWorkingDir, captured.BundlePath) } +// A bundle path in or below a project, by the walk every deploy decides by, +// is refused: a 1.x project's Dockerfile beside .astro (with no +// .astro/config.yaml), and a pyproject.toml project. func (s *NonDagsDeploySuite) TestWithinAstroProject() { - projectDir, cleanup, err := config.CreateTempProject() - assert.NoError(s.T(), err) - defer cleanup() - - err = testExecCmd(NewDeployCmd(), "test-deployment-id", "--non-dags", "--non-dags-mount-path", "/usr/local/airflow/x", "--non-dags-bundle-type", "dbt", "--non-dags-local-path", projectDir) - assert.Error(s.T(), err) - assert.Contains(s.T(), err.Error(), "within an Astro project") + oneX := s.T().TempDir() + s.Require().NoError(os.WriteFile(filepath.Join(oneX, "Dockerfile"), []byte("FROM x\n"), 0o600)) + s.Require().NoError(os.MkdirAll(filepath.Join(oneX, ".astro"), 0o755)) + manifestDir := s.T().TempDir() + s.Require().NoError(os.WriteFile(filepath.Join(manifestDir, "pyproject.toml"), []byte("[project]\nname = \"demo\"\n\n[tool.astro]\n"), 0o600)) + s.Require().NoError(os.MkdirAll(filepath.Join(manifestDir, "include", "dbt"), 0o755)) + + for _, bundle := range []string{oneX, filepath.Join(manifestDir, "include", "dbt")} { + resetDeployFlagVars() + err := testExecCmd(NewDeployCmd(), "test-deployment-id", "--non-dags", "--non-dags-mount-path", "/usr/local/airflow/x", "--non-dags-bundle-type", "dbt", "--non-dags-local-path", bundle) + s.Require().Error(err, bundle) + s.Contains(err.Error(), "within an Astro project", bundle) + } } func (s *NonDagsDeploySuite) TestBundlePathDoesNotExist() { @@ -379,3 +367,92 @@ func (s *NonDagsDeploySuite) TestBundlePathNotADirectory() { assert.Error(s.T(), err) assert.Contains(s.T(), err.Error(), "is not a directory") } + +// --deployment names a non-Dag bundle's Deployment as the argument does. +func (s *NonDagsDeploySuite) TestDeploymentFlag() { + var captured *astrodeploy.DeployBundleInput + DeployBundle = func(deployInput *astrodeploy.DeployBundleInput) (astrodeploy.BundleDeploy, error) { + captured = deployInput + return astrodeploy.BundleDeploy{}, nil + } + + err := testExecCmd(NewDeployCmd(), "--deployment", "test-deployment-id", "--non-dags", "--non-dags-mount-path", "/usr/local/airflow/x") + assert.NoError(s.T(), err) + s.Require().NotNil(captured) + assert.Equal(s.T(), "test-deployment-id", captured.DeploymentID) +} + +// --non-dags runs before the project decides anything: from a pyproject.toml +// project it deploys a bundle, not the project, so the project directory +// itself is refused as the bundle. It used to run a whole project deploy, +// dropping --non-dags. +func (s *NonDagsDeploySuite) TestFromAManifestProject() { + s.Require().NoError(os.WriteFile(filepath.Join(s.tmpWorkingDir, "pyproject.toml"), []byte("[project]\nname = \"demo\"\n\n[tool.astro]\n"), 0o600)) + DeployBundle = func(*astrodeploy.DeployBundleInput) (astrodeploy.BundleDeploy, error) { + s.Fail("a bundle inside the project is not deployed") + return astrodeploy.BundleDeploy{}, nil + } + + err := testExecCmd(NewDeployCmd(), "test-deployment-id", "--non-dags", "--non-dags-mount-path", "/usr/local/airflow/x") + assert.ErrorContains(s.T(), err, "within an Astro project") +} + +// --output json publishes the bundle deploy as one object; a bad --output is +// a usage error before anything is read. +func (s *NonDagsDeploySuite) TestOutput() { + DeployBundle = func(in *astrodeploy.DeployBundleInput) (astrodeploy.BundleDeploy, error) { + return astrodeploy.BundleDeploy{DeploymentID: in.DeploymentID, DeployID: "dep-1", MountPath: in.MountPath, BundleVersion: "v1"}, nil + } + resetDeployFlagVars() + out, err := execDeployCapture("test-deployment-id", "--non-dags", "--non-dags-mount-path", "/usr/local/airflow/x", "--output", "json") + s.Require().NoError(err, out) + var got nonDagsDeployJSON + s.Require().NoError(json.Unmarshal([]byte(out), &got), out) + s.Equal(nonDagsDeployJSON{Deployment: "test-deployment-id", DeployID: "dep-1", BundleType: "none", BundlePath: s.tmpWorkingDir, MountPath: "/usr/local/airflow/x", BundleVersion: "v1"}, got) + + resetDeployFlagVars() + _, err = execDeployCapture("test-deployment-id", "--non-dags", "--non-dags-mount-path", "/usr/local/airflow/x", "--output", "yaml") + s.True(cliout.IsUsage(err), "%v", err) +} + +// The target is named as a project deploy names it: the argument and +// --deployment must agree, and a link name of the project here is its +// Deployment. --workspace is the workspace whose Deployments are offered. +func (s *NonDagsDeploySuite) TestTarget() { + var captured *astrodeploy.DeployBundleInput + DeployBundle = func(in *astrodeploy.DeployBundleInput) (astrodeploy.BundleDeploy, error) { + captured = in + return astrodeploy.BundleDeploy{}, nil + } + bundle := s.T().TempDir() + + resetDeployFlagVars() + err := testExecCmd(NewDeployCmd(), "dep-a", "--deployment", "dep-b", "--non-dags", "--non-dags-mount-path", "/x", "--non-dags-local-path", bundle) + s.ErrorContains(err, "name one") + s.True(cliout.IsUsage(err)) + + s.Require().NoError(os.WriteFile(filepath.Join(s.tmpWorkingDir, "pyproject.toml"), []byte("[project]\nname = \"demo\"\ndependencies = [\"apache-airflow==3.1.*\"]\n\n[tool.astro]\n\n[tool.astro.deployments.prod]\ntarget = \"astro\"\nworkspace = \"clw-ws\"\ndeployment = \"clx-prod\"\n"), 0o600)) + resetDeployFlagVars() + err = testExecCmd(NewDeployCmd(), "prod", "--non-dags", "--non-dags-mount-path", "/x", "--non-dags-local-path", bundle) + s.Require().NoError(err) + s.Equal("clx-prod", captured.DeploymentID) + s.Nil(captured.Login, "a project naming no host deploys under the current context") +} + +// A Deployment id needs no manifest: a pyproject.toml here that does not load +// does not stop a deploy that names its target by id. +func (s *NonDagsDeploySuite) TestIDNeedsNoManifest() { + var captured *astrodeploy.DeployBundleInput + DeployBundle = func(in *astrodeploy.DeployBundleInput) (astrodeploy.BundleDeploy, error) { + captured = in + return astrodeploy.BundleDeploy{}, nil + } + s.Require().NoError(os.WriteFile(filepath.Join(s.tmpWorkingDir, "pyproject.toml"), []byte("[tool.astro]\nruntime = [\n"), 0o600)) + bundle := s.T().TempDir() + + resetDeployFlagVars() + err := testExecCmd(NewDeployCmd(), "--deployment", "clx-by-id", "--non-dags", "--non-dags-mount-path", "/x", "--non-dags-local-path", bundle) + s.Require().NoError(err) + s.Equal("clx-by-id", captured.DeploymentID) + s.Nil(captured.Login) +} diff --git a/cmd/astro/deployment.go b/cmd/astro/deployment.go index 1f61eccc5..4fab1353d 100644 --- a/cmd/astro/deployment.go +++ b/cmd/astro/deployment.go @@ -39,6 +39,8 @@ var ( label string runtimeVersion string deploymentID string + deploymentName string + envFile string logsKeyword string forceDelete bool description string diff --git a/cmd/astro/main_test.go b/cmd/astro/main_test.go index c4baf73d3..2f165cc06 100644 --- a/cmd/astro/main_test.go +++ b/cmd/astro/main_test.go @@ -27,10 +27,6 @@ func TestMain(m *testing.M) { flag.Parse() origArgs := os.Args os.Args = []string{"astro"} - // Deploy refuses a checkout with uncommitted changes, and these tests run - // in a real one: without this, they would pass or fail with the developer's - // working tree. A test of the refusal sets its own. - hasUncommittedChanges = func(string) bool { return false } // A login reads its auth config from api., which no unit test may // reach. The identity provider it names is under .test, which never // resolves, so a refresh against it fails without leaving the machine, as diff --git a/cmd/astro/output_binding_test.go b/cmd/astro/output_binding_test.go index e24bbbfe3..630f6a493 100644 --- a/cmd/astro/output_binding_test.go +++ b/cmd/astro/output_binding_test.go @@ -4,6 +4,7 @@ import ( "context" "io" "os" + "os/exec" "path/filepath" "testing" @@ -84,6 +85,33 @@ func TestJSONResultReachesStdoutUnderAnUnboundRoot(t *testing.T) { assert.Equal(t, "removed", got.Action) assert.Empty(t, stderr) }) + // The bundle sits in a git checkout with uncommitted changes, so the + // platform code prints its note on the way: under json that goes to + // stderr, and stdout is the one result object. + t.Run("deploy --non-dags from a dirty checkout", func(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("needs git") + } + bundle := dbtProject(t) + gitIn := func(args ...string) { + t.Helper() + full := append([]string{"-C", bundle, "-c", "user.name=t", "-c", "user.email=t@example.invalid", "-c", "commit.gpgsign=false"}, args...) + out, err := exec.Command("git", full...).CombinedOutput() + require.NoError(t, err, string(out)) + } + gitIn("init", "-q") + gitIn("add", ".") + gitIn("commit", "-q", "-m", "first") + require.NoError(t, os.WriteFile(filepath.Join(bundle, "dbt_project.yml"), []byte("name: changed\n"), 0o600)) + resetDeployFlagVars() + + stdout, stderr, err := execUnboundRoot(t, dbtBundleMock(t, fakeBlobStore(t)), "deploy", dbtTestDeploymentID, "--non-dags", "--non-dags-mount-path", "/usr/local/airflow/x", "--non-dags-local-path", bundle, "--output", "json") + require.NoError(t, err, "stderr:\n%s", stderr) + var got nonDagsDeployJSON + decodeOne(t, stdout, &got) + assert.Equal(t, dbtTestVersion, got.BundleVersion) + assert.Contains(t, stderr, "uncommitted changes", "the platform's note goes to stderr") + }) t.Run("remote deploy", func(t *testing.T) { stubClientDeploy(t, &astrodeploy.ClientDeploy{Image: "r/c:deploy-x", Registry: "r/c", Tag: "deploy-x"}, nil) dir := t.TempDir() diff --git a/cmd/astro/remote.go b/cmd/astro/remote.go index 409264f31..ad6441124 100644 --- a/cmd/astro/remote.go +++ b/cmd/astro/remote.go @@ -10,7 +10,6 @@ import ( "github.com/astronomer/astro-cli/cmd/utils" "github.com/astronomer/astro-cli/config" astrodeploy "github.com/astronomer/astro-cli/internal/platform/astro/deploy" - "github.com/astronomer/astro-cli/internal/project" "github.com/astronomer/astro-cli/pkg/util" ) @@ -54,14 +53,19 @@ func newRemoteDeployCmd() *cobra.Command { Short: "Deploy a client image to the remote registry", Long: "Build and deploy a client image to the configured remote registry. This command assumes you have already authenticated with the registry.", PreRunE: func(cmd *cobra.Command, args []string) error { - // A project with a pyproject.toml has no .astro/config.yaml, so the 1.x check would - // reject it. astro deploy grew this bypass and its sibling here - // never did, which left remote deploy refusing every such project - // with advice to run astro dev init, a command v2 removed. - if project.HasManifest(config.WorkingPath) { + // The rule astro deploy follows: a build needs a pyproject.toml + // project's root, and in or below a project in the Astro CLI 1.x + // layout, or below a pyproject.toml project's root, everything is + // refused with the same advice. --image-name builds nothing, so + // outside any project it pushes the image as it is. The value, + // not whether the flag was given: an empty --image-name= names + // no image. + where := utils.Locate(config.WorkingPath) + if where.AtRoot(config.WorkingPath) || (remoteImageName != "" && where.None()) { return nil } - return utils.EnsureProjectDir(cmd, args) + cmd.SilenceUsage = true + return utils.NoDeployableProject(where, utils.Deploy1xRefusedAstro) }, RunE: remoteDeploy, Example: remoteDeployExample, diff --git a/cmd/astro/remote_test.go b/cmd/astro/remote_test.go index e30f75068..18ecde596 100644 --- a/cmd/astro/remote_test.go +++ b/cmd/astro/remote_test.go @@ -2,10 +2,13 @@ package astro import ( "os" + "path/filepath" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/astronomer/astro-cli/config" testUtil "github.com/astronomer/astro-cli/pkg/testing" ) @@ -271,3 +274,33 @@ func TestRemoteCommandIntegration(t *testing.T) { assert.Contains(t, deployCmd.Example, "checking its runtime against a Deployment") }) } + +// remote deploy follows astro deploy's rule: building needs a pyproject.toml +// project, a project in the Astro CLI 1.x layout is refused with the same +// advice, and --image-name, which builds nothing, runs anywhere. +func TestRemoteDeployProjectCheck(t *testing.T) { + testUtil.InitTestConfig(testUtil.LocalPlatform) + prevPath, prevImage := config.WorkingPath, remoteImageName + t.Cleanup(func() { config.WorkingPath, remoteImageName = prevPath, prevImage }) + + check := func(dir string, args ...string) error { + config.WorkingPath = dir + remoteImageName = "" + cmd := newRemoteDeployCmd() + require.NoError(t, cmd.ParseFlags(args)) + return cmd.PreRunE(cmd, nil) + } + + empty := t.TempDir() + assert.ErrorContains(t, check(empty), "not an Astro project directory") + assert.NoError(t, check(empty, "--image-name", "img")) + assert.Error(t, check(empty, "--image-name="), "an empty --image-name= names no image") + + oneX := make1xProject(t) + assert.ErrorContains(t, check(oneX), "uses the Astro CLI 1.x layout") + assert.ErrorContains(t, check(oneX, "--image-name", "img"), "uses the Astro CLI 1.x layout", "a 1.x checkout is refused, --image-name included") + + manifestProject := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(manifestProject, "pyproject.toml"), []byte("[project]\nname = \"demo\"\n\n[tool.astro]\n"), 0o600)) + assert.NoError(t, check(manifestProject)) +} diff --git a/cmd/astro/schema_test.go b/cmd/astro/schema_test.go index 5f4642ea6..f235e8e6e 100644 --- a/cmd/astro/schema_test.go +++ b/cmd/astro/schema_test.go @@ -153,6 +153,7 @@ var publishedPayloads = []cliouttest.Case{ // astro deploy --output json, in a project with a manifest: the one // object a finished deploy prints. {Name: "deploy", Value: deployJSON{}}, + {Name: "deploy-non-dags", Value: nonDagsDeployJSON{}}, // astro dbt deploy: what it deployed where. delete: the bundle it // removed. cleanup: the artifacts it removed and kept. {Name: "dbt-deploy", Value: dbtDeployJSON{}}, diff --git a/cmd/astro/testdata/schema/deploy-non-dags.json b/cmd/astro/testdata/schema/deploy-non-dags.json new file mode 100644 index 000000000..4b06085e0 --- /dev/null +++ b/cmd/astro/testdata/schema/deploy-non-dags.json @@ -0,0 +1,17 @@ +{ + "deployment": "x", + "deployment_name": "x", + "workspace": "x", + "deploy_id": "x", + "bundle_type": "x", + "bundle_path": "x", + "mount_path": "x", + "bundle_version": "x", + "waited": true, + "wait_error": "x", + "git": { + "commit_sha": "x", + "branch": "x", + "commit_url": "x" + } +} diff --git a/cmd/astro/testdata/schema/deploy.json b/cmd/astro/testdata/schema/deploy.json index acd253078..e464fbe65 100644 --- a/cmd/astro/testdata/schema/deploy.json +++ b/cmd/astro/testdata/schema/deploy.json @@ -11,5 +11,8 @@ "commit_sha": "x", "branch": "x", "commit_url": "x" - } + }, + "warnings": [ + "x" + ] } diff --git a/cmd/config.go b/cmd/config.go index c0477c5bd..3d9aa3f56 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -43,6 +43,9 @@ var removedConfigKeys = map[string]string{ "without it a free port is chosen.", "webserver.port": "Pick the local Airflow port with `astro local start --port `; " + "without it a free port is chosen.", + // astro deploy --save wrote it into a 1.x project's .astro/config.yaml. + "project.deployment": "Name the Deployment on each deploy, as the argument or with --deployment. " + + "On Astro, link a project's Deployments with astro link add, and mark one default = true to preselect it.", } // refuseRemovedConfigKey fails a set of a removed key the way the removed @@ -171,9 +174,10 @@ func ensureGlobalFlag(cmd *cobra.Command, args []string) error { if len(args) == 0 { return nil } - // Ahead of the scope checks, so the answer is the same in and out of a - // project and with or without -g. - if cmd.Name() == "set" { + // A removed key is refused by set and get alike. Ahead of the scope + // checks, so the answer is the same in and out of a project and with or + // without -g. + if cmd.Name() == "set" || cmd.Name() == "get" { if err := refuseRemovedConfigKey(args[0]); err != nil { cmd.SilenceUsage = true return err @@ -281,7 +285,8 @@ func configSet(cmd *cobra.Command, args []string) error { func configList(r cliout.Renderer) error { list := configSettings{Settings: []configSetting{}} for _, key := range slices.Sorted(maps.Keys(config.CFGStrMap)) { - if unlistedConfigs[key] { + // The removed keys too: get and set refuse them, so list agrees. + if _, removed := removedConfigKeys[key]; unlistedConfigs[key] || removed { continue } cfg := config.CFGStrMap[key] diff --git a/cmd/config_output_test.go b/cmd/config_output_test.go index d9bbe7005..8bbf9a0db 100644 --- a/cmd/config_output_test.go +++ b/cmd/config_output_test.go @@ -137,6 +137,10 @@ func (s *CmdSuite) TestConfigListOutput() { for key := range unlistedConfigs { s.NotContains(keys, key) } + // What get and set refuse is not listed either, so list and get agree. + for key := range removedConfigKeys { + s.NotContains(keys, key) + } s.NotContains(stdout, "t0ken-value") }) diff --git a/cmd/config_test.go b/cmd/config_test.go index 85dd37446..420fc9eb1 100644 --- a/cmd/config_test.go +++ b/cmd/config_test.go @@ -191,6 +191,7 @@ func (s *CmdSuite) TestConfigSetRefusesRemovedKeys() { {"proxy.port", "6599", "listens on 6563"}, {"api-server.port", "8099", "astro local start --port "}, {"webserver.port", "8099", "astro local start --port "}, + {"project.deployment", "clx-old", "as the argument or with --deployment"}, } { for _, scope := range []string{"outside a project", "global", "manifest project"} { s.Run(tc.key+" "+scope, func() { @@ -217,6 +218,17 @@ func (s *CmdSuite) TestConfigSetRefusesRemovedKeys() { } } +// get refuses a removed key as set does, rather than show a value nothing +// reads. +func (s *CmdSuite) TestConfigGetRefusesRemovedKeys() { + testUtil.InitTestConfig(testUtil.LocalPlatform) + s.useWorkingPath(s.T().TempDir()) + _, err := executeCommand("config", "get", "project.deployment") + s.Require().Error(err) + s.True(cliout.IsUsage(err), "want a usage error, got %v", err) + s.ErrorContains(err, "`project.deployment` was removed in Astro CLI v2") +} + // configSet guards the write on its own, for a caller that skips the pre-run. func (s *CmdSuite) TestConfigSetNeverWritesARemovedKey() { testUtil.InitTestConfig(testUtil.LocalPlatform) diff --git a/cmd/removed_flags.go b/cmd/removed_flags.go index c32f9bd67..5ac90abd1 100644 --- a/cmd/removed_flags.go +++ b/cmd/removed_flags.go @@ -38,6 +38,22 @@ const ( // link to. errLoginLinkRemoved = "--login-link was removed in Astro CLI v2: switching organizations no longer re-authenticates. " + "To log in on another device, use `astro login --login-link`" + + // astro deploy ships only pyproject.toml projects in v2, so the flags only + // the 1.x project's deploy read went with it. A 1.x project deploys with + // Astro CLI 1.x, flags and all. + errDeploySaveRemoved = "--save was removed in Astro CLI v2: astro deploy asks which Deployment to deploy to unless you name one, as the argument or with --deployment. " + + "To preselect one in that list, link it with astro link add and mark it default = true under [tool.astro.deployments]" + errAPCDeploySaveRemoved = "--save was removed in Astro CLI v2: name the Deployment on each deploy, as the argument" + errDeployPytestRemoved = "--pytest was removed in Astro CLI v2: run your tests before deploying, with uv run pytest && astro deploy" + errDeployTestRemoved = "--test was removed in Astro CLI v2: run your tests before deploying, with uv run pytest && astro deploy" + errDeployEnvRemoved = "--env was removed in Astro CLI v2: astro deploy runs no tests, so it reads no test env file. Run your tests with uv run pytest before deploying" + errDeployParseRemoved = "--parse was removed in Astro CLI v2: check your DAGs before deploying, with astro local check && astro deploy" + errDeployDagsPathRemoved = "--dags-path was removed in Astro CLI v2: astro deploy --dags ships the project's dags directory, so run it from the project" + errDeployBundleNameRemoved = "--dag-bundle-name was removed in Astro CLI v2: deploying to a named DAG bundle is not supported yet. Use Astro CLI 1.x for it" + errDeployNameRemoved = "--deployment-name was removed in Astro CLI v2: name the Deployment as the argument or with --deployment, which take a link name or a Deployment id" + errAPCDeployNoCache = "--no-cache was removed in Astro CLI v2: Astro CLI v2 builds no image for Astro Private Cloud. " + + "Build and deploy the project with Astro CLI 1.x" ) // removedFlag is what a run passing a flag Astro CLI 1.x had and v2 dropped @@ -104,6 +120,18 @@ var removedFlags = []removedFlag{ {name: "format", needs: cliout.HasOutput, msgFor: formatRemoved}, // --force skipped the confirmation that --yes skips now. {name: "force", needs: has("yes"), msgFor: forceRemoved}, + // astro deploy's 1.x-project flags. Astro's deploy has --deployment for + // --save to point at; APC's names its Deployment by the argument alone. + {name: "save", under: []string{"deploy"}, needs: has("deployment"), msg: errDeploySaveRemoved}, + {name: "save", under: []string{"deploy"}, msg: errAPCDeploySaveRemoved}, + {name: "pytest", under: []string{"deploy"}, msg: errDeployPytestRemoved}, + {name: "test", under: []string{"deploy"}, msg: errDeployTestRemoved}, + {name: "env", under: []string{"deploy"}, msg: errDeployEnvRemoved}, + {name: "parse", under: []string{"deploy"}, msg: errDeployParseRemoved}, + {name: "dags-path", under: []string{"deploy"}, msg: errDeployDagsPathRemoved}, + {name: "dag-bundle-name", under: []string{"deploy"}, msg: errDeployBundleNameRemoved}, + {name: "deployment-name", under: []string{"deploy"}, msg: errDeployNameRemoved}, + {name: "no-cache", under: []string{"deploy"}, msg: errAPCDeployNoCache}, } // removedFlagErr is a run refused for passing a removed 1.x flag. diff --git a/cmd/removed_flags_test.go b/cmd/removed_flags_test.go index 91095f6c6..31fa058bc 100644 --- a/cmd/removed_flags_test.go +++ b/cmd/removed_flags_test.go @@ -111,6 +111,16 @@ var removedFlagCases = []removedFlagCase{ {tree: astroTree, args: []string{"deployment", "delete", "dep-id"}, flag: "force", shorthand: "f", want: "--force was removed in Astro CLI v2: use --yes (-y)"}, {tree: astroHostedTree, args: []string{"deployment", "bundle", "delete"}, flag: "force", shorthand: "f", want: "--force was removed in Astro CLI v2: use --yes (-y)"}, {tree: apcTree, args: []string{"deployment", "update", "dep-id"}, flag: "force", shorthand: "f", want: "--force was removed in Astro CLI v2: use --yes (-y)"}, + {tree: astroTree, args: []string{"deploy", "dep-id"}, flag: "save", shorthand: "s", want: errDeploySaveRemoved}, + {tree: apcTree, args: []string{"deploy", "dep-id"}, flag: "save", shorthand: "s", want: errAPCDeploySaveRemoved}, + {tree: astroTree, args: []string{"deploy", "dep-id"}, flag: "pytest", want: errDeployPytestRemoved}, + {tree: astroHostedTree, args: []string{"deploy", "dep-id"}, flag: "test", value: "tests/test_dags.py", shorthand: "t", want: errDeployTestRemoved}, + {tree: astroTree, args: []string{"deploy", "dep-id"}, flag: "env", value: ".env.ci", shorthand: "e", want: errDeployEnvRemoved}, + {tree: astroTree, args: []string{"deploy", "dep-id"}, flag: "parse", want: errDeployParseRemoved}, + {tree: astroTree, args: []string{"deploy", "dep-id", "--dags"}, flag: "dags-path", value: "./elsewhere", want: errDeployDagsPathRemoved}, + {tree: astroTree, args: []string{"deploy", "dep-id"}, flag: "dag-bundle-name", value: "nightly", want: errDeployBundleNameRemoved}, + {tree: astroTree, args: []string{"deploy"}, flag: "deployment-name", value: "prod", shorthand: "n", want: errDeployNameRemoved}, + {tree: apcTree, args: []string{"deploy", "dep-id"}, flag: "no-cache", want: errAPCDeployNoCache}, } // spellings is every way the case's flag can be typed: --name, --name=value diff --git a/cmd/utils/utils.go b/cmd/utils/utils.go index fda7bc09e..9f650f063 100644 --- a/cmd/utils/utils.go +++ b/cmd/utils/utils.go @@ -1,11 +1,18 @@ package utils import ( - "github.com/pkg/errors" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "regexp" + "github.com/spf13/cobra" "github.com/astronomer/astro-cli/config" - "github.com/astronomer/astro-cli/pkg/ansi" + "github.com/astronomer/astro-cli/internal/project" + "github.com/astronomer/astro-cli/pkg/manifest" ) type RunE func(cmd *cobra.Command, args []string) error @@ -22,34 +29,235 @@ func ChainRunEs(runEs ...RunE) RunE { } } -// EnsureProjectDir fails outside a project directory, with advice to run astro -// init. Its callers on Astro accept the pyproject.toml project astro init writes -// before they get here, so the advice leads somewhere. -func EnsureProjectDir(cmd *cobra.Command, args []string) error { - return ensureProjectDir("Change to an Astro project directory, or run astro init to make this one an Astro project") +// Refusal1x is what a deploy says when the working directory is in a +// project in the Astro CLI 1.x layout, at dir: where is "this project" when +// the deploy ran in it, and names dir when it ran below it. +type Refusal1x func(where, dir string) string + +// What a deploy from a project in the Astro CLI 1.x layout is told. v2 +// deploys only pyproject.toml projects, and Astro CLI 1.x keeps deploying the +// 1.x layout, so nobody has to convert a project to keep shipping it. +// +// On Astro Private Cloud the advice is Astro CLI 1.x alone: v2 builds no +// project there yet, and astro init refuses to convert a project under an +// APC context, to keep its 1.x deploys working. +var ( + Deploy1xRefusedAstro Refusal1x = func(where, dir string) string { + return where + " uses the Astro CLI 1.x layout (a Dockerfile and .astro/config.yaml), and Astro CLI v2 deploys only pyproject.toml projects. " + + "Convert it with astro init" + in(dir) + ", or deploy it with Astro CLI 1.x" + } + Deploy1xRefusedAPC Refusal1x = func(where, _ string) string { + return where + " uses the Astro CLI 1.x layout (a Dockerfile and .astro/config.yaml), which Astro CLI v2 does not deploy to Astro Private Cloud. " + + "Deploy it with Astro CLI 1.x" + } +) + +func in(dir string) string { + if dir == "" { + return "" + } + return " in " + dir +} + +// The no-project advice: the home directory, where astro init would make +// every file under ~ part of a project, and anywhere else. +const ( + homeDirRefusal = "this is your home directory, not an Astro project directory. Change to an Astro project directory" + notProjectAdvice = "this is not an Astro project directory. Change to an Astro project directory, or run astro init to make this one an Astro project" +) + +// Where is what a deploy finds at and above its working directory: the +// nearest pyproject.toml project (ManifestDir) or project in the Astro CLI +// 1.x layout (Project1xDir), whichever comes first; both empty for none. +// +// For a pyproject.toml project it carries the manifest the walk loaded, or +// why it did not load (LoadErr: it fails to parse or validate, or cannot be +// read), so nothing after the walk reads the file again. +type Where struct { + ManifestDir string + Project1xDir string + Manifest *manifest.Manifest + LoadErr error +} + +// AtRoot reports whether the walk from dir found a pyproject.toml project at +// dir itself: the deploy runs at the project's root. +func (w Where) AtRoot(dir string) bool { + return w.ManifestDir != "" && sameDir(w.ManifestDir, dir) } -// EnsureDockerfileProjectDir is EnsureProjectDir for APC deploy, which builds a -// Dockerfile project and has no path for the pyproject.toml project astro init -// writes, so advice to run astro init would only lead back here. Such a project -// reaches APC through its image instead: with a declared Dockerfile, astro -// package builds the whole project (the runtime base's ONBUILD COPY bakes the -// DAGs in) and tags an image carrying the runtime label --image-name requires. -func EnsureDockerfileProjectDir(cmd *cobra.Command, args []string) error { - return ensureProjectDir("Deploying to APC needs a Dockerfile-based project, one with a .astro/config.yaml, such as a project made with Astro CLI 1.x. From a pyproject.toml project, declare dockerfile = \"Dockerfile\" under [tool.astro] (a Dockerfile FROM an Astro Runtime image), run astro package, and deploy the image it tags with --image-name") +// None reports whether the walk found no project at all. +func (w Where) None() bool { return w.ManifestDir == "" && w.Project1xDir == "" } + +// Loaded is the manifest of a project the deploy runs at the root of, nil +// when it runs anywhere else or the manifest did not load. +func (w Where) Loaded(dir string) *manifest.Manifest { + if !w.AtRoot(dir) { + return nil + } + return w.Manifest } -func ensureProjectDir(advice string) error { - isProjectDir, err := config.IsProjectDir(config.WorkingPath) +// Locate walks up from dir to the nearest project, the one rule astro +// deploy, astro remote deploy, APC's deploy and the bundle deploys' +// containment checks decide by, routing included. At each directory: +// +// - a pyproject.toml project's root (manifestRoot) stops the walk. A +// pyproject.toml declaring no tool.astro table, a monorepo root's tool +// settings say, does not; +// - else a Dockerfile beside a .astro directory is a 1.x project, except +// in the home directory, whose .astro holds the global config; +// - else the walk goes on up. +// +// Each directory's pyproject.toml is read once, and the home directory is +// looked up once per walk. +func Locate(dir string) Where { + abs, err := filepath.Abs(dir) if err != nil { - return errors.Wrap(err, ansi.Red("failed to verify that your working directory is an Astro project.\n"+advice)) + return Where{} + } + isHome := homeMatcher() + for d := abs; ; { + if root, m, loadErr := manifestRoot(d); root { + return Where{ManifestDir: d, Manifest: m, LoadErr: loadErr} + } + // Not a manifest root, so the 1.x layout alone decides: no second + // read of the pyproject.toml (project.Is1xProject would make one). + if has1xLayout(d) && !isHome(d) { + return Where{Project1xDir: d} + } + parent := filepath.Dir(d) + if parent == d { + return Where{} + } + d = parent + } +} + +// homeMatcher reports whether a directory is the home directory, with +// config.HomePath looked up once: the same spelling once cleaned, or the +// same directory under another one (a symlink), as config.IsHomeDir has it. +func homeMatcher() func(dir string) bool { + if config.HomePath == "" { + return func(string) bool { return false } + } + home := filepath.Clean(config.HomePath) + homeInfo, homeErr := os.Stat(home) + return func(dir string) bool { + if filepath.Clean(dir) == home { + return true + } + if homeErr != nil { + return false + } + info, err := os.Stat(dir) + return err == nil && os.SameFile(info, homeInfo) + } +} + +// has1xLayout reports whether dir holds a Dockerfile beside a .astro +// directory. A directory that cannot be read holds neither. +func has1xLayout(dir string) bool { + if _, err := os.Stat(filepath.Join(dir, "Dockerfile")); err != nil { + return false + } + info, err := os.Stat(filepath.Join(dir, ".astro")) + return err == nil && info.IsDir() +} + +// manifestRoot reports whether dir is a pyproject.toml project's root, the +// question every deploy routes on: +// +// - a pyproject.toml whose [tool.astro] loads, or fails to validate: a +// project to fix, and the deploy reports why; +// - one that fails to parse, only when its text declares a tool.astro +// table (a [tool.astro header, or a tool.astro key): otherwise it is +// some other tool's file, and not a project; +// - one that exists and cannot be read, as project.HasManifest has it: the +// deploy then reports the read error rather than walking past it. +// +// No pyproject.toml, one with no [tool.astro], and one in a directory that +// cannot be looked in are not. It returns the manifest it loaded, or why a +// root's manifest did not load. +func manifestRoot(dir string) (root bool, m *manifest.Manifest, loadErr error) { + path := filepath.Join(dir, project.Marker) + if info, err := os.Stat(path); err != nil || !info.Mode().IsRegular() { + // Not there, or a directory it cannot even look in: nothing at this + // level, and the walk goes on. + return false, nil, nil + } + m, err := manifest.Load(path) + switch { + case err == nil: + return true, m, nil + case errors.Is(err, manifest.ErrNotFound), errors.Is(err, manifest.ErrNoAstroSection), errors.Is(err, fs.ErrNotExist): + return false, nil, nil } + raw, readErr := os.ReadFile(path) + if readErr != nil { + // There, and unreadable. + return !errors.Is(readErr, fs.ErrNotExist), nil, err + } + if declaresToolAstro.Match(raw) { + return true, nil, err + } + return false, nil, nil +} - if !isProjectDir { - return errors.New(ansi.Red("this is not an Astro project directory.\n" + advice + "\n")) +// declaresToolAstro matches a TOML line declaring the tool.astro table: a +// [tool.astro] or [tool.astro.] header, an array-of-tables header under +// it, or a dotted tool.astro key. +var declaresToolAstro = regexp.MustCompile(`(?m)^\s*(\[\[?\s*"?tool"?\s*\.\s*"?astro"?\s*[\].]|"?tool"?\s*\.\s*"?astro"?\s*[.=])`) + +// NoDeployableProject is what a deploy says when the working directory is not +// the root of a pyproject.toml project, from Locate's answer where (for the +// working directory): +// +// - in or below a project in the Astro CLI 1.x layout, refused1x, naming +// that project's directory when the deploy ran below it; +// - below a pyproject.toml project, to run the deploy from its root; +// - anywhere else, the no-project advice, which in the home directory does +// not suggest astro init. +// +// Every one of these is reported under the kind no_project: it unwraps to a +// *project.NotFoundError. +func NoDeployableProject(where Where, refused1x Refusal1x) error { + wd := config.WorkingPath + switch { + case where.Project1xDir != "": + says, dir := "this project", "" + if !sameDir(where.Project1xDir, wd) { + says, dir = "this directory is inside a project at "+where.Project1xDir+" that", where.Project1xDir + } + return &noProjectError{msg: refused1x(says, dir), cause: &project.NotFoundError{Start: wd, Project1xDir: where.Project1xDir}} + case where.ManifestDir != "" && !sameDir(where.ManifestDir, wd): + return NoProject(fmt.Sprintf("this directory is inside the project at %s. Run the deploy from the project directory, %s", where.ManifestDir, where.ManifestDir)) + case config.IsHomeDir(wd): + return NoProject(homeDirRefusal) } + return NoProject(notProjectAdvice) +} + +func sameDir(a, b string) bool { + absA, errA := filepath.Abs(a) + absB, errB := filepath.Abs(b) + return errA == nil && errB == nil && filepath.Clean(absA) == filepath.Clean(absB) +} + +// noProjectError carries its own words, and unwraps to the +// *project.NotFoundError that cmd/local's ProblemKinds reports as no_project. +type noProjectError struct { + msg string + cause error +} + +func (e *noProjectError) Error() string { return e.msg } +func (e *noProjectError) Unwrap() error { return e.cause } - return nil +// NoProject is a refusal for want of a project here, saying msg, reported +// under the kind no_project. +func NoProject(msg string) error { + return &noProjectError{msg: msg, cause: &project.NotFoundError{Start: config.WorkingPath}} } func GetDefaultDeployDescription(isDagOnlyDeploy bool) string { diff --git a/cmd/utils/utils_test.go b/cmd/utils/utils_test.go index 1cec73930..03fa16727 100644 --- a/cmd/utils/utils_test.go +++ b/cmd/utils/utils_test.go @@ -2,54 +2,244 @@ package utils import ( "errors" + "os" + "path/filepath" + "runtime" "testing" "github.com/spf13/cobra" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/astronomer/astro-cli/config" + "github.com/astronomer/astro-cli/internal/project" ) -func TestEnsureProjectDir(t *testing.T) { - currentWorkingPath := config.WorkingPath - fileName := config.ConfigFileNameWithExt - dirName := config.ConfigDir - defer func() { - config.WorkingPath = currentWorkingPath - config.ConfigFileNameWithExt = fileName - config.ConfigDir = dirName - }() - // error case when file path is not resolvable - config.WorkingPath = "./\000x" - err := EnsureProjectDir(&cobra.Command{}, []string{}) - assert.Error(t, err) - assert.Contains(t, err.Error(), "failed to verify that your working directory is an Astro project.\nChange to an Astro project directory, or run astro init to make this one an Astro project") +func writeFile(t *testing.T, dir, name, content string) { + t.Helper() + require.NoError(t, os.MkdirAll(filepath.Dir(filepath.Join(dir, name)), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(content), 0o600)) +} - // error case when no such file or dir - config.WorkingPath = "./test" - err = EnsureProjectDir(&cobra.Command{}, []string{}) - assert.Error(t, err) - assert.Contains(t, err.Error(), "this is not an Astro project directory.\nChange to an Astro project directory, or run astro init to make this one an Astro project") - // astro dev init does not exist in v2 - assert.NotContains(t, err.Error(), "dev init") +// make1x lays out a project the way Astro CLI 1.x made one. +func make1x(t *testing.T, dir string) { + t.Helper() + writeFile(t, dir, "Dockerfile", "FROM quay.io/astronomer/astro-runtime:12.0.0\n") + writeFile(t, dir, filepath.Join(".astro", "config.yaml"), "project:\n name: demo\n") +} - // APC deploy cannot use the project astro init writes, so it is not told to run it - err = EnsureDockerfileProjectDir(&cobra.Command{}, []string{}) - assert.Error(t, err) - assert.Contains(t, err.Error(), "this is not an Astro project directory.\nDeploying to APC needs a Dockerfile-based project, one with a .astro/config.yaml") - assert.NotContains(t, err.Error(), "astro init") - // it points a pyproject.toml project at the path that does reach APC - assert.Contains(t, err.Error(), `declare dockerfile = "Dockerfile" under [tool.astro]`) - assert.Contains(t, err.Error(), "run astro package, and deploy the image it tags with --image-name") - - // success case - config.WorkingPath = currentWorkingPath - config.ConfigFileNameWithExt = "utils_test.go" - config.ConfigDir = "" - err = EnsureProjectDir(&cobra.Command{}, []string{}) - assert.NoError(t, err) - err = EnsureDockerfileProjectDir(&cobra.Command{}, []string{}) - assert.NoError(t, err) +func deployIn(t *testing.T, dir string) { + t.Helper() + prev := config.WorkingPath + config.WorkingPath = dir + t.Cleanup(func() { config.WorkingPath = prev }) +} + +// isNoProject reports whether err is reported under the kind no_project. +func isNoProject(err error) bool { + var notFound *project.NotFoundError + return errors.As(err, ¬Found) +} + +// A deploy outside a pyproject.toml project gets one of three answers, all +// no_project, by project.Discover's rule. +func TestNoDeployableProject(t *testing.T) { + prevHome := config.HomePath + t.Cleanup(func() { config.HomePath = prevHome }) + config.HomePath = t.TempDir() + + t.Run("a 1.x project", func(t *testing.T) { + dir := t.TempDir() + make1x(t, dir) + deployIn(t, dir) + err := refuse(Deploy1xRefusedAstro) + assert.EqualError(t, err, "this project uses the Astro CLI 1.x layout (a Dockerfile and .astro/config.yaml), and Astro CLI v2 deploys only pyproject.toml projects. Convert it with astro init, or deploy it with Astro CLI 1.x") + assert.True(t, isNoProject(err)) + }) + + t.Run("below a 1.x project, which it names", func(t *testing.T) { + dir := t.TempDir() + make1x(t, dir) + sub := filepath.Join(dir, "dags") + require.NoError(t, os.MkdirAll(sub, 0o755)) + deployIn(t, sub) + err := refuse(Deploy1xRefusedAstro) + require.Error(t, err) + assert.Contains(t, err.Error(), "this directory is inside a project at "+dir+" that uses the Astro CLI 1.x layout") + assert.Contains(t, err.Error(), "Convert it with astro init in "+dir+",") + assert.True(t, isNoProject(err)) + + err = refuse(Deploy1xRefusedAPC) + assert.Contains(t, err.Error(), "which Astro CLI v2 does not deploy to Astro Private Cloud. Deploy it with Astro CLI 1.x") + assert.NotContains(t, err.Error(), "astro init") + }) + + t.Run("a 1.x project keeping a pyproject.toml for its tools", func(t *testing.T) { + dir := t.TempDir() + make1x(t, dir) + writeFile(t, dir, "pyproject.toml", "[tool.ruff]\nline-length = 100\n") + deployIn(t, dir) + err := refuse(Deploy1xRefusedAstro) + assert.Contains(t, err.Error(), "this project uses the Astro CLI 1.x layout") + }) + + t.Run("below a pyproject.toml project", func(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "pyproject.toml", "[project]\nname = \"demo\"\n\n[tool.astro]\n") + sub := filepath.Join(dir, "dags") + require.NoError(t, os.MkdirAll(sub, 0o755)) + deployIn(t, sub) + err := refuse(Deploy1xRefusedAstro) + assert.EqualError(t, err, "this directory is inside the project at "+dir+". Run the deploy from the project directory, "+dir) + assert.True(t, isNoProject(err)) + }) + + t.Run("no project", func(t *testing.T) { + deployIn(t, t.TempDir()) + err := refuse(Deploy1xRefusedAstro) + assert.EqualError(t, err, notProjectAdvice) + assert.True(t, isNoProject(err)) + }) + + t.Run("a .astro/config.yaml alone is no project", func(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, filepath.Join(".astro", "config.yaml"), "project:\n name: demo\n") + deployIn(t, dir) + assert.EqualError(t, refuse(Deploy1xRefusedAstro), notProjectAdvice) + }) +} + +// In the home directory the advice does not suggest astro init, which would +// make all of ~ a project, and the home directory is recognized under another +// spelling too. A Dockerfile beside ~/.astro, where the global config lives, +// does not make it a 1.x project, nor does a tooling-only pyproject.toml. +func TestNoDeployableProjectInTheHomeDirectory(t *testing.T) { + prevHome := config.HomePath + t.Cleanup(func() { config.HomePath = prevHome }) + home := t.TempDir() + config.HomePath = home + make1x(t, home) + writeFile(t, home, "pyproject.toml", "[tool.ruff]\nline-length = 100\n") + for _, dir := range []string{home, home + string(filepath.Separator), filepath.Join(home, ".")} { + deployIn(t, dir) + err := refuse(Deploy1xRefusedAstro) + assert.EqualError(t, err, homeDirRefusal) + assert.True(t, isNoProject(err)) + assert.Equal(t, Where{}, Locate(dir)) + } + + link := filepath.Join(t.TempDir(), "home") + if err := os.Symlink(home, link); err == nil { + deployIn(t, link) + assert.EqualError(t, refuse(Deploy1xRefusedAstro), homeDirRefusal) + } +} + +// Locate's walk: the nearest pyproject.toml with [tool.astro] or 1.x project, +// a tooling-only pyproject.toml passed over. +func TestLocate(t *testing.T) { + t.Run("a pyproject.toml project is never a 1.x one", func(t *testing.T) { + dir := t.TempDir() + make1x(t, dir) + writeFile(t, dir, "pyproject.toml", "[project]\nname = \"demo\"\n\n[tool.astro]\n") + assert.Equal(t, dir, Locate(dir).ManifestDir) + w := Locate(dir) + assert.True(t, w.Manifest != nil || w.LoadErr != nil, "the walk carries what loading the manifest gave") + }) + + t.Run("the walk carries the manifest it loaded, read once", func(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "pyproject.toml", "[project]\nname = \"demo\"\ndependencies = [\"apache-airflow==3.1.*\"]\n\n[tool.astro]\n") + w := Locate(dir) + require.NoError(t, w.LoadErr) + require.NotNil(t, w.Manifest) + assert.Same(t, w.Manifest, w.Loaded(dir)) + assert.Nil(t, w.Loaded(t.TempDir()), "only at the root") + }) + + t.Run("a monorepo's tooling pyproject.toml above a 1.x project", func(t *testing.T) { + root := t.TempDir() + writeFile(t, root, "pyproject.toml", "[tool.ruff]\nline-length = 100\n") + oneX := filepath.Join(root, "airflow") + make1x(t, oneX) + assert.Equal(t, Where{Project1xDir: oneX}, Locate(oneX)) + sub := filepath.Join(oneX, "dags", "team") + require.NoError(t, os.MkdirAll(sub, 0o755)) + assert.Equal(t, Where{Project1xDir: oneX}, Locate(sub)) + deployIn(t, oneX) + assert.Contains(t, refuse(Deploy1xRefusedAstro).Error(), "this project uses the Astro CLI 1.x layout") + }) + + t.Run("directory names that are not ASCII", func(t *testing.T) { + dir := filepath.Join(t.TempDir(), "проект-データ") + make1x(t, dir) + assert.Equal(t, Where{Project1xDir: dir}, Locate(dir)) + other := filepath.Join(t.TempDir(), "日本") + require.NoError(t, os.MkdirAll(other, 0o755)) + writeFile(t, other, "pyproject.toml", "[project]\nname = \"demo\"\n\n[tool.astro]\n") + assert.Equal(t, other, Locate(other).ManifestDir) + }) + + t.Run("an unreadable pyproject.toml is a root, whose deploy reports why", func(t *testing.T) { + if runtime.GOOS == "windows" || os.Geteuid() == 0 { + t.Skip("file modes do not stop this reader") + } + dir := t.TempDir() + writeFile(t, dir, "pyproject.toml", "[project]\nname = \"demo\"\n\n[tool.astro]\n") + require.NoError(t, os.Chmod(filepath.Join(dir, "pyproject.toml"), 0o000)) + t.Cleanup(func() { _ = os.Chmod(filepath.Join(dir, "pyproject.toml"), 0o600) }) + assert.True(t, isRoot(dir)) + assert.Equal(t, project.HasManifest(dir), isRoot(dir), "routing and the walk agree with project.HasManifest") + sub := filepath.Join(dir, "dags") + require.NoError(t, os.MkdirAll(sub, 0o755)) + assert.Equal(t, dir, Locate(sub).ManifestDir) + }) + + t.Run("a directory that cannot be looked in is passed over", func(t *testing.T) { + if runtime.GOOS == "windows" || os.Geteuid() == 0 { + t.Skip("directory modes do not stop this reader") + } + top := t.TempDir() + locked := filepath.Join(top, "locked") + writeFile(t, locked, "pyproject.toml", "[project]\nname = \"demo\"\n\n[tool.astro]\n") + inner := filepath.Join(locked, "inner") + require.NoError(t, os.MkdirAll(inner, 0o755)) + require.NoError(t, os.Chmod(locked, 0o000)) + t.Cleanup(func() { _ = os.Chmod(locked, 0o755) }) + assert.Equal(t, Where{}, Locate(inner)) + }) + + t.Run("a pyproject.toml that fails to parse", func(t *testing.T) { + // Declaring tool.astro: a project to fix, and the deploy says why. + for _, text := range []string{"[tool.astro]\nruntime = [\n", "[tool.astro.env]\nX = {\n", "tool.astro.runtime = [\n"} { + dir := t.TempDir() + writeFile(t, dir, "pyproject.toml", text) + assert.True(t, isRoot(dir), "%q", text) + assert.Equal(t, dir, Locate(dir).ManifestDir, "%q", text) + assert.Error(t, Locate(dir).LoadErr, "%q: why it did not load, for the deploy to report", text) + } + // Some other tool's broken file: not a root, and the walk goes on + // to the 1.x project below it. + root := t.TempDir() + writeFile(t, root, "pyproject.toml", "[tool.ruff\nline-length = 100\n") + assert.False(t, isRoot(root)) + oneX := filepath.Join(root, "airflow") + make1x(t, oneX) + sub := filepath.Join(oneX, "dags") + require.NoError(t, os.MkdirAll(sub, 0o755)) + assert.Equal(t, Where{Project1xDir: oneX}, Locate(sub)) + }) +} + +// isRoot is manifestRoot's answer alone. +func isRoot(dir string) bool { + root, _, _ := manifestRoot(dir) + return root +} + +// refuse is NoDeployableProject for the working directory, as a deploy calls it. +func refuse(r Refusal1x) error { + return NoDeployableProject(Locate(config.WorkingPath), r) } func TestGetDefaultDeployDescription(t *testing.T) { diff --git a/config/config.go b/config/config.go index 6eed649f6..65be615d5 100644 --- a/config/config.go +++ b/config/config.go @@ -7,7 +7,6 @@ import ( iofs "io/fs" "os" "path/filepath" - "strings" "time" "github.com/gofrs/flock" @@ -262,31 +261,26 @@ func IsProjectDir(path string) (bool, error) { configFile := filepath.Join(configPath, ConfigFileNameWithExt) // Home directory is not a project directory - if HomePath == path { + if IsHomeDir(path) { return false, nil } return fileutil.Exists(configFile, nil) } -// IsWithinProjectDir returns true if the path is at or within an Astro project directory -func IsWithinProjectDir(path string) (bool, error) { - pathAbs, err := filepath.Abs(filepath.Clean(path)) - if err != nil { - return false, err +// IsHomeDir reports whether path is HomePath: the same spelling once cleaned, +// or, when both can be read, the same directory under another one (a trailing +// slash, a symlink). +func IsHomeDir(path string) bool { + if path == "" || HomePath == "" { + return false } - pathComponents := strings.Split(pathAbs, string(os.PathSeparator)) - for i := range pathComponents { - componentAbs := strings.Join(pathComponents[:i+1], string(os.PathSeparator)) - isProjectDir, err := IsProjectDir(componentAbs) - if err != nil { - return false, err - } - if isProjectDir { - return true, nil - } + if filepath.Clean(path) == filepath.Clean(HomePath) { + return true } - return false, nil + a, errA := os.Stat(path) + b, errB := os.Stat(HomePath) + return errA == nil && errB == nil && os.SameFile(a, b) } // saveConfig serializes config writes under an exclusive OS-level file lock, diff --git a/config/config_test.go b/config/config_test.go index 927deae67..dca2d2d08 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -42,32 +42,32 @@ func (s *Suite) TestIsProjectDir() { } } -func (s *Suite) TestIsWithinProjectDir() { +// The home directory is never a project directory, under any spelling of it, +// even with a .astro/config.yaml in it (the global config lives there). +func (s *Suite) TestIsProjectDirRefusesHomeUnderAnySpelling() { + prev := HomePath + defer func() { HomePath = prev }() + HomePath = s.T().TempDir() + s.Require().NoError(os.MkdirAll(filepath.Join(HomePath, ConfigDir), 0o755)) + s.Require().NoError(os.WriteFile(filepath.Join(HomePath, ConfigDir, ConfigFileNameWithExt), nil, 0o600)) + paths := []string{HomePath, HomePath + string(os.PathSeparator)} + // Windows makes a symlink only with a privilege CI may not grant. + if link := filepath.Join(s.T().TempDir(), "home-link"); os.Symlink(HomePath, link) == nil { + paths = append(paths, link) + } + + for _, path := range paths { + got, err := IsProjectDir(path) + s.NoError(err) + s.False(got, path) + } + projectDir, cleanupProjectDir, err := CreateTempProject() - s.NoError(err) + s.Require().NoError(err) defer cleanupProjectDir() - - anotherDir, err := os.MkdirTemp("", "") + got, err := IsProjectDir(projectDir) s.NoError(err) - - tests := []struct { - name string - in string - out bool - }{ - {"not in", anotherDir, false}, - {"at", projectDir, true}, - {"just in", filepath.Join(projectDir, "test"), true}, - {"deep in", filepath.Join(projectDir, "test", "test", "test"), true}, - {"root", string(os.PathSeparator), false}, - } - for _, tt := range tests { - s.Run(tt.name, func() { - got, err := IsWithinProjectDir(tt.in) - s.NoError(err) - s.Equal(got, tt.out) - }) - } + s.True(got) } func (s *Suite) TestInitHomeDefaultCase() { diff --git a/docker/parse.go b/docker/parse.go index 1d4521897..886325904 100644 --- a/docker/parse.go +++ b/docker/parse.go @@ -6,7 +6,6 @@ import ( "os" "strings" - "github.com/distribution/reference" "github.com/moby/buildkit/frontend/dockerfile/command" "github.com/moby/buildkit/frontend/dockerfile/parser" ) @@ -99,35 +98,3 @@ func GetImageFromParsedFile(cmds []Command) (image string) { } return "" } - -// Parse tag from parsed dockerfile: -// e.g. FROM ubuntu:xenial returns "ubuntu", "xenial" -func GetImageTagFromParsedFile(cmds []Command) (baseImage, tag string) { - for _, cmd := range cmds { - if strings.EqualFold(cmd.Cmd, command.From) { - if len(cmd.Value) > 0 { - from := cmd.Value[0] - baseImage, tag := parseImageName(from) - return baseImage, tag - } - } - } - return "", "" -} - -func parseImageName(imageName string) (baseImage, tag string) { - ref, err := reference.Parse(imageName) - if err != nil { - return baseImage, tag - } - parsedName, ok := ref.(reference.Named) - if ok { - baseImage = parsedName.Name() - } - tag = "latest" - parsedTag, ok := ref.(reference.Tagged) - if ok { - tag = parsedTag.Tag() - } - return baseImage, tag -} diff --git a/docker/parse_test.go b/docker/parse_test.go index 5a4914979..d8a98fa2a 100644 --- a/docker/parse_test.go +++ b/docker/parse_test.go @@ -120,84 +120,3 @@ func (s *Suite) TestGetImageFromParsedFile() { s.Equal("", image) }) } - -func (s *Suite) TestGetImageTagFromParsedFile() { - s.Run("success with real parser output", func() { - dockerfile := `FROM quay.io/astronomer/astro-runtime:3.0.2` - cmds, err := ParseReader(bytes.NewBufferString(dockerfile)) - s.NoError(err) - - image, tag := GetImageTagFromParsedFile(cmds) - s.Equal("quay.io/astronomer/astro-runtime", image) - s.Equal("3.0.2", tag) - }) - - s.Run("success with artificial lowercase cmd (legacy)", func() { - cmds := []Command{ - { - Cmd: "from", - Original: "FROM quay.io/astronomer/astro-runtime:3.0.2", - StartLine: 1, - EndLine: 1, - Flags: []string{}, - Value: []string{"quay.io/astronomer/astro-runtime:3.0.2"}, - }, - } - image, tag := GetImageTagFromParsedFile(cmds) - s.Equal("quay.io/astronomer/astro-runtime", image) - s.Equal("3.0.2", tag) - }) - - s.Run("no image name found", func() { - cmds := []Command{ - { - Cmd: "echo", - Original: "echo $?", - StartLine: 1, - EndLine: 1, - Flags: []string{}, - Value: []string{"$?"}, - }, - } - image, tag := GetImageTagFromParsedFile(cmds) - s.Equal("", image) - s.Equal("", tag) - }) - - s.Run("empty value array should not panic", func() { - cmds := []Command{ - { - Cmd: "FROM", - Original: "FROM", - StartLine: 1, - EndLine: 1, - Flags: []string{}, - Value: []string{}, // Empty value array - this would panic without bounds check - }, - } - // This should return empty strings, not panic - image, tag := GetImageTagFromParsedFile(cmds) - s.Equal("", image) - s.Equal("", tag) - }) -} - -func (s *Suite) TestParseImageName() { - tests := []struct { - imageName string - expectedBaseImage string - expectedTag string - }{ - {imageName: "localhost:5000/airflow:latest", expectedBaseImage: "localhost:5000/airflow", expectedTag: "latest"}, - {imageName: "localhost/airflow:1.2.0", expectedBaseImage: "localhost/airflow", expectedTag: "1.2.0"}, - {imageName: "quay.io:5000/airflow", expectedBaseImage: "quay.io:5000/airflow", expectedTag: "latest"}, - {imageName: "airflow:latest", expectedBaseImage: "airflow", expectedTag: "latest"}, - {imageName: "airflow", expectedBaseImage: "airflow", expectedTag: "latest"}, - } - - for _, tt := range tests { - baseImage, tag := parseImageName(tt.imageName) - s.Equal(tt.expectedBaseImage, baseImage) - s.Equal(tt.expectedTag, tag) - } -} diff --git a/docs/architecture.md b/docs/architecture.md index fe7856fa4..2e40c00c4 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -160,7 +160,7 @@ A command that already refuses to ask when it has no terminal marks that refusal | status | meaning | | --- | --- | | 0 | success | -| 1 | the command failed, including `input_required`: a question it could not ask. Not 2, because a question can come after work has started (a 1.x project's DAG deploy asks about an empty `dags/` after it has created the deploy record), so the "nothing ran" that 2 promises would not hold. The same refusal without a terminal has always exited 1. A script that needs to tell it from other failures reads `kind` | +| 1 | the command failed, including `input_required`: a question it could not ask. Not 2, because a question can come after work has started, so the "nothing ran" that 2 promises would not hold. The same refusal without a terminal has always exited 1. A script that needs to tell it from other failures reads `kind` | | 2 | usage error: an unknown flag or subcommand, a flag value the command does not accept (`--output yaml` included), the wrong number of arguments, a missing required flag, a group run with no subcommand under `--output json`, whether it has a RunE of its own (`cliout.GroupHelp`) or not (`cliout.Execute` answers it); in text mode it prints its help and exits 0, as before. Nothing ran | | 130 | interrupted (Ctrl-C or SIGTERM); the command unwound what it had started | | the command's own | `astro local check`: 1 when a check failed, 2 when it could not reach a verdict (no environment, no project). `astro af runs trigger-wait` (and `astro local af runs trigger-wait`): 1 when the run failed, 2 when the wait timed out. `astro local run`: the child's status | diff --git a/docs/deploy.md b/docs/deploy.md index 63b4b599d..91429c045 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -8,11 +8,48 @@ Terms used below: - The **transport** is the set of calls that talk to Astro: create the deploy, push the image to the Deployment's registry, upload the DAG tarball, finalize. - The **runtime image** is Astronomer's Airflow base image (`astrocrpublic.azurecr.io/runtime`). Building `FROM` it runs its `ONBUILD` steps, which install the project's `requirements.txt` and `packages.txt`. -## Routing: a project or a 1.x project +## What deploys: a `pyproject.toml` project, or an image already built -`astro deploy` looks at the working directory. A `pyproject.toml` with a `[tool.astro]` table takes the manifest path, even if the table fails to validate (the deploy then reports the manifest error). Anything else takes the 1.x path, `deploy.Deploy` in `internal/platform/astro/deploy`, which is unchanged: it deploys a project made by Astro CLI 1.x (a 1.x project). A Dockerfile is not evidence of a 1.x project: a project with a manifest may declare one (`[tool.astro] dockerfile`). The check is `HasManifest` in [`internal/project`](../internal/project/project.go), and the fork is in [`cmd/astro/deploy.go`](../cmd/astro/deploy.go). +v2 builds and deploys only projects with a `pyproject.toml`, on Astro and on Astro Private Cloud. Nobody has to move to v2, and Astro CLI 1.x keeps deploying the projects it made, so v2 has no path for the 1.x layout (a `Dockerfile` and `.astro/config.yaml`). -The manifest path lives beside the 1.x path in the same package (`manifest.go`, `manifest_image.go`) and in [`internal/deploy`](../internal/deploy/deploy.go). It calls the 1.x path's transport pieces directly instead of wrapping its `Deploy()`, which prints, rewrites `.dockerignore` and assumes the 1.x layout. +`astro deploy` looks at the working directory. A `pyproject.toml` with a `[tool.astro]` table is deployed, even if the table fails to validate (the deploy then reports the manifest error). A Dockerfile is not evidence of a 1.x project: a project with a manifest may declare one (`[tool.astro] dockerfile`). The routing is in [`cmd/astro/deploy.go`](../cmd/astro/deploy.go). + +Anywhere else, one walk decides (`utils.Locate`, used by `astro deploy`, `astro remote deploy`, APC's deploy, and the bundle-path checks of `--non-dags` and `astro dbt deploy`). From the working directory up, at each directory: a `pyproject.toml` with `[tool.astro]` is a project's root; else a Dockerfile beside a `.astro` directory is a 1.x project, except in the home directory, whose `.astro` holds the global config; else the walk goes on up. A `pyproject.toml` that declares no `tool.astro` table (a monorepo root's `[tool.ruff]`, even one that fails to parse) does not stop it. One that declares it stops the walk even when it fails to parse or validate, and so does one that exists and cannot be read, so the deploy reports why; a directory that cannot be looked in holds neither. The deploy routes on the same walk, and uses the manifest it loaded rather than reading the file again. + +| the walk finds | `astro deploy`, any mode but `--non-dags` | +| --- | --- | +| a 1.x project, here or above | refused: `this project uses the Astro CLI 1.x layout (a Dockerfile and .astro/config.yaml), and Astro CLI v2 deploys only pyproject.toml projects. Convert it with astro init, or deploy it with Astro CLI 1.x`. From below it, it names the project: `this directory is inside a project at that uses …. Convert it with astro init in , …` | +| a `pyproject.toml` project above | refused: `this directory is inside the project at . Run the deploy from the project directory, ` | +| nothing | `--image-name` deploys (below); anything else is refused: `this is not an Astro project directory. Change to an Astro project directory, or run astro init to make this one an Astro project` (in the home directory, only the advice to change directory) | + +Every refusal is reported under `--output json` as the error object with kind `no_project`, and exit 1. `--image-name` is refused in and below a 1.x project like every other mode: a prebuilt image deployed from a 1.x checkout would leave that project's DAGs stale with nothing said. + +**`--image-name ` outside any project** deploys an image already built on this machine, alone: an image-only deploy, with no `dags/` and no git commit. The target is the argument or `--deployment`, a Deployment id (there are no links without a manifest), or the workspace's Deployments to pick from; a run that cannot ask says to pass the id as the argument or with `--deployment`. The login is the current context's. The deploy warns, on stderr and in the result's `warnings`: a Deployment that takes DAG deploys keeps the DAGs it had, and one that does not now runs the DAGs inside the image, if any. At a `pyproject.toml` project's root, `--image-name` is the project's deploy with a prebuilt image, `dags/` included as usual. + +**`--non-dags`** ships a separate directory as a non-DAG bundle, which must not be inside any project, so it is checked first and runs anywhere. A bundle path in or below any project the walk finds is refused, as `astro dbt deploy` refuses a dbt project there: the two use the same check. + +- **The login** is chosen as `astro deploy ` from the same directory chooses it. In a `pyproject.toml` project here whose manifest loads, the project's login (for its Astro host) is used for everything: Deployment ids, link names and the workspace picker. Anywhere else, or when the manifest does not load, the current context is used for everything. +- **The target** is the argument or `--deployment`, which must agree: a link of the project here, or a Deployment id. With neither, the Deployments of `--workspace` (or `--workspace-id`, or the context's) are offered, but only to a run that can be asked and only on the current context's host. A run under `--output json` or without a terminal fails with kind `input_required` and `pass the Deployment id as the argument or with --deployment`, as `astro deploy` does. A project on another host is told to pass `--deployment ` or switch context. +- **Output:** `--output json` publishes one object, pinned by `cmd/astro/testdata/schema/deploy-non-dags.json`. What the deploy prints on the way (the git note, for one) goes to stderr there. + +**Flags that cannot all apply** are refused as usage errors before anything is read: `--wait-time` without `--wait`, `--no-dags-base-dir` with a deploy that ships no DAGs (`--image`, or `--image-name` outside a project), and `--dags` with `--image` or `--image-name` (the message names the ones given). + +`astro remote deploy`, which builds and pushes a Remote Execution client image, follows the same walk: it builds at a `pyproject.toml` project's root, refuses in or below a 1.x project and below a project's root with the messages above, and with `--image-name` pushes an image already built from outside any project. + +The deploy code lives in `internal/platform/astro/deploy` (`manifest.go`, `manifest_image.go`, and the bundle and Remote Execution client paths beside them) and in [`internal/deploy`](../internal/deploy/deploy.go). + +### Astro Private Cloud + +v2 does not build projects for Astro Private Cloud yet. Support for `pyproject.toml` projects there is planned. Until then, `astro deploy` under an APC context follows the same walk and the same `--image-name` rule: + +- **a deploy that would build the project**, one without `--image-name`, is refused: `Astro CLI v2 cannot build and deploy projects to Astro Private Cloud yet: use Astro CLI 1.x for now.` At a `pyproject.toml` project's root it adds that support is coming, and that `--dags` uploads the project's DAGs and an image built with `astro package` deploys with `--image-name`; that is a usage error (exit 2), a mode v2 does not have. Outside any project it is kind `no_project` (exit 1), as on Astro; +- **in or below a project in the 1.x layout every deploy is refused**, `--image-name` and `--dags` included: `… uses the Astro CLI 1.x layout (a Dockerfile and .astro/config.yaml), which Astro CLI v2 does not deploy to Astro Private Cloud. Deploy it with Astro CLI 1.x` (kind `no_project`). It does not suggest `astro init`, which refuses to convert a project under an APC context. Below a `pyproject.toml` project's root, the deploy is told to run from the root; +- **`--image-name `** (with `--remote`, an image already in a registry) deploys at a project's root, where the project's `dags/` then goes to a Deployment that takes DAG uploads, and outside any project, where the image ships alone: no `dags/` is uploaded, and when the Deployment takes DAG uploads (or what it takes could not be read) the deploy says its DAGs were not updated, whatever `show_warnings` is; +- **`--dags`** uploads a `pyproject.toml` project's DAGs, from the project's root; with `--image-name` it is refused, in `astro deploy`'s words: `--dags deploys only your DAGs; drop --image-name`. + +APC's uncommitted-changes check (`--force` passes it) holds only for a deploy at a project's root: an image deployed alone from outside any project reads nothing from the directory. + +The target is the argument, or the Deployment picked from the workspace's. The `project.deployment` a 1.x project's `.astro/config.yaml` saved (`--save`) is not read, and `astro config set project.deployment` says it was removed. ## 1. Image deploys @@ -60,7 +97,7 @@ The manifest's `[tool.astro.deployments.]` links name the Deployments a pr - **The argument or `--deployment` names a link or an id.** `astro deploy prod` ships to the `prod` link. A value no link declares is used as a Deployment id, so `astro deploy ` (what astronomer/deploy-action runs) keeps working. Naming both with different values is an error. There is no `-d` shorthand for `--deployment`: on this command `-d` means `--dags`. - **With nothing named, deploy asks.** The prompt (on stderr) is the CLI's numbered picker table of the project's Astro links. `ASTRO_DEPLOYMENT`, then your `astro use` selection, then the `default = true` link preselect the highlighted entry, labelled in its `PRESELECTED BY` column with whichever put it there (`ASTRO_DEPLOYMENT`, `astro use`, `default = true`), and the prompt shows its number (`> [2]`). Enter takes it. None of them can skip the question, and the prompt does not change your `astro use` selection. A project with exactly one link preselects it as its default. An answer is a link name or a number, exactly as listed, names first, so a link called `2` selects itself. Three bad answers give up. - **A run that cannot be asked fails.** With no terminal or with `--output json`, a deploy that names nothing stops: `a deploy must name the deployment it ships to: astro deploy or --deployment . …`. Under `--output json` it is the error object with kind `input_required` (see [Prompts](architecture.md#prompts)). -- **A project that links nothing** needs a workspace (`--workspace`, else the current context), then runs the 1.x path's workspace flow: pick a Deployment, or create one (`deployment.GetDeployment`). A run that cannot be asked must pass `--deployment`. +- **A project that links nothing** needs a workspace (`--workspace`, else the current context), then runs the workspace flow `astro deployment` commands use: pick a Deployment, or create one (`deployment.GetDeployment`). A run that cannot be asked must pass `--deployment`. - **A project whose links are all non-Astro** is refused, naming what it declares, rather than offered an unrelated Deployment. **Workspace.** `--workspace`, else the link's workspace (its own, or the `[tool.astro]` default it inherits), else the current context's. `--workspace-id`, the older spelling, still works and is hidden from help; given with `--workspace`, the two must agree. @@ -169,7 +206,7 @@ astro package [TARGET] [flags] # astro (default), mwaa, composer, oss -o, --output text|json ``` -The 1.x-only flags (`--save`, `--pytest`, `--env`, `--test`, `--parse`, `--deployment-name`, `--dags-path`, `--dag-bundle-name`) are refused on the manifest path with the alternative to use. `--force` and `--prompt` are accepted and do nothing. +The flags only a 1.x project's deploy read are gone: `--save` (`-s`), `--pytest`, `--env` (`-e`), `--test` (`-t`), `--parse`, `--deployment-name` (`-n`), `--dags-path` and `--dag-bundle-name`, and on Astro Private Cloud `--save` and `--no-cache`. A run passing one fails as a usage error naming what replaced it (run the tests with `uv run pytest` first, check the DAGs with `astro local check`, name the Deployment with `--deployment`, and so on; see [Removed 1.x commands and flags](architecture.md#removed-1x-commands-and-flags)). `--force` (`-f`) and `--prompt` (`-p`) are accepted, hidden and read by nothing: astronomer/deploy-action passes `--force` on every deploy, and the deploy always asks unless a Deployment is named, which is what `--prompt` asked for. `--non-dags` and its `--non-dags-*` flags deploy a non-DAG bundle from a separate directory (see above). **Git metadata.** With the `deploy.git_metadata` setting on (the default), the deploy records the HEAD commit on Astro, and the commit message becomes the description when `--description` is not given. `commit_url` is set only for a GitHub remote. A project with uncommitted changes to tracked files records no commit and prints `note: the project has uncommitted changes, so this deploy records no git commit` on stderr. `--image-name` records none. diff --git a/docs/install.md b/docs/install.md index 854c67cce..91783e69d 100644 --- a/docs/install.md +++ b/docs/install.md @@ -22,6 +22,10 @@ removes only the files of a project made by Astro CLI 1.x (a 1.x project) whose contents it carried across, saying so. So everything below shows up as a `git diff` you can read before you commit it. +Astro CLI v2 deploys only a converted project. A 1.x project you have not +converted keeps deploying with Astro CLI 1.x, and so does any project on Astro +Private Cloud for now: v2 does not build and deploy projects there yet. + ## Two things to ask about first If you are an agent following this page, stop and ask the person before you do diff --git a/docs/upgrading-from-v1.md b/docs/upgrading-from-v1.md index 60ed9c313..5e4c3f498 100644 --- a/docs/upgrading-from-v1.md +++ b/docs/upgrading-from-v1.md @@ -2,6 +2,18 @@ This page is for people who use Astro CLI 1.x today. It covers what v2 does differently, then what breaks. Old commands and flags fail with a message that names the v2 replacement, so an old script tells you what to change instead of failing silently. +## v2 requires converting your project + +v2 works on projects with a `pyproject.toml`, and only on those. It does not run, start or deploy a project in the 1.x layout (a `Dockerfile` and `.astro/config.yaml`). You don't have to move: Astro CLI 1.x keeps working, and keeps deploying the projects it made. When you do move, convert each project once: + +```sh +astro init +``` + +`astro init` converts a 1.x project in place (see [A project is a `pyproject.toml`](#a-project-is-a-pyprojecttoml) and [install.md](install.md#step-4-check-the-conversion-and-finish-it)). Until a project is converted, `astro deploy` in it fails with `this project uses the Astro CLI 1.x layout …, and Astro CLI v2 deploys only pyproject.toml projects. Convert it with astro init, or deploy it with Astro CLI 1.x`, and `astro dev` commands point at `astro init` too. `astro deploy --image-name` too: a prebuilt image deployed from a 1.x checkout would leave its DAGs stale. Outside any project, `--image-name` deploys an image you already built, alone, with no DAGs. + +On Astro Private Cloud, v2 does not build and deploy projects yet, converted or not: use Astro CLI 1.x to deploy there for now. v2 deploys an image you built there (`astro deploy --image-name`, from a converted project or outside any project), and uploads a converted project's DAGs (`astro deploy --dags`). Support for `pyproject.toml` projects on Astro Private Cloud is coming. + ## What's new ### A project is a `pyproject.toml` @@ -43,17 +55,17 @@ A project can link the Deployments it ships to, by name, in `pyproject.toml`. `a ### Deploying and packaging -`astro deploy` builds from the manifest, and fails fast on a runtime the Deployment can't take, before the build rather than after it. `astro package` builds the same artifact without shipping it, for Astro, MWAA or Cloud Composer. A 1.x project with no `[tool.astro]` still deploys the 1.x way, unchanged. See [deploy.md](deploy.md). +`astro deploy` builds from the manifest, and fails fast on a runtime the Deployment can't take, before the build rather than after it. `astro package` builds the same artifact without shipping it, for Astro, MWAA or Cloud Composer. A 1.x project is not deployed until it is converted (see [v2 requires converting your project](#v2-requires-converting-your-project)). See [deploy.md](deploy.md). ### Scripting: `-o json` Commands that print a result take `-o json` (`--output json`), with a stable shape for scripts, agents and Astro Desktop. Under `-o json`, results go to stdout and progress and prompts go to stderr. The exceptions: -- `astro deploy` takes only the long `--output json`, and only in a project with a `pyproject.toml`; +- `astro deploy` on Astro takes only the long `--output json`; - `astro login`, `astro logout` and `astro otto` are interactive or print nothing worth parsing, and have no `-o`; - requests through `astro api airflow`, `cloud` and `registry` print the API's own response, shaped with `--jq` and `--template` (their `ls` and `describe` take `-o`); - `astro completion` prints a shell script. -Commands that ask before deleting or replacing something take `--yes` (`-y`) to answer for you. `astro deploy` has no `--yes`: in a `pyproject.toml` project it asks only which Deployment to deploy to, which naming one (`--deployment`, or a link name as the argument) answers, and in a 1.x project it asks what 1.x asked: a Deployment id as the argument skips the choice of Deployment, and `--force` answers the warnings. +Commands that ask before deleting or replacing something take `--yes` (`-y`) to answer for you. `astro deploy` on Astro has no `--yes`: it asks only which Deployment to deploy to, which naming one (`--deployment`, or a link name as the argument) answers. ### Smaller changes @@ -99,7 +111,7 @@ Run inside a 1.x project, an `astro dev` command also tells you to convert it wi | 1.x | v2 | where | | --- | --- | --- | -| `--force`, `-f` | `--yes`, `-y` | the 1.x commands that took it and ask for confirmation now (deployment, bundle, token, team, worker-queue and context commands). `astro deploy --force` is unchanged | +| `--force`, `-f` | `--yes`, `-y` | the 1.x commands that took it and ask for confirmation now (deployment, bundle, token, team, worker-queue and context commands). `astro deploy` on Astro still accepts `--force` and `--prompt`, hidden, and reads nothing from them | | `--json` | `-o json` | `list` commands, `astro api … ls` / `describe` | | `--template` | `-o json`, and jq | `list` commands | | `-o table`, `-o template`, `-o yaml` | `-o text` (the default) or `-o json` | everywhere `-o` existed, except `astro deployment inspect`, which keeps `-o yaml`. These get the general "unknown output format" error, which lists the formats the command takes | @@ -110,14 +122,22 @@ Run inside a 1.x project, an `astro dev` command also tells you to convert it wi | `-o ` | `--output-file ` (`-o` is the output format now) | `astro organization audit-logs export` | | `--api-url` | `--url` | `astro api airflow`, and its `ls` / `describe` | | `--deployment-id` | `--deployment`, `-d` | `astro api airflow`, and its `ls` / `describe` | +| `--pytest`, `--test` (`-t`), `--env` (`-e`) | run your tests first: `uv run pytest && astro deploy` | `astro deploy` | +| `--parse` | check your DAGs first: `astro local check && astro deploy` | `astro deploy` | +| `--deployment-name`, `-n` | the argument, or `--deployment`, either taking a link name or a Deployment id | `astro deploy` | +| `--save`, `-s` | name the Deployment on each deploy; on Astro, link it with `astro link add` and mark it `default = true` to preselect it | `astro deploy` | +| `--dags-path` | `astro deploy --dags` from the project, which ships its `dags/` | `astro deploy` | +| `--dag-bundle-name` | no replacement yet: use Astro CLI 1.x | `astro deploy` | +| `--no-cache` | no replacement: v2 builds no image for Astro Private Cloud | `astro deploy` on Astro Private Cloud | -Some 1.x flags still work but are hidden from help in favor of a new spelling. On the `astro deployment` subcommands, `astro env` and `astro dbt deploy` / `delete`, `--deployment-name` (`-n`) and `--deployment-id`, whichever the command has, give way to `--deployment`. `--workspace-id` gives way to `--workspace` on those commands and the other commands that have both, except `astro deploy`, which shows both. `astro deploy` also still shows `-n`, `--deployment-name`: it works in a 1.x project, and a project with a `pyproject.toml` refuses it in favor of `--deployment`. `astro deploy --dags-path` is hidden, and works only in a 1.x project. +Some 1.x flags still work but are hidden from help in favor of a new spelling. On the `astro deployment` subcommands, `astro env` and `astro dbt deploy` / `delete`, `--deployment-name` (`-n`) and `--deployment-id`, whichever the command has, give way to `--deployment`. `--workspace-id` gives way to `--workspace` on those commands and the other commands that have both, `astro deploy` among them. ### Behavior +- **`project.deployment` is gone.** `astro deploy --save` wrote it into a 1.x project's `.astro/config.yaml`, and nothing in v2 reads it: `astro config set` and `get` of it say it was removed. Name the Deployment on each deploy, as the argument or with `--deployment`, or link it with `astro link add`. - **An `astro api airflow` request needs a target.** 1.x defaulted to `localhost:8080`. v2 needs `-d ` or `--url`. For the Airflow on your machine, use `astro local api`. `ls` and `describe` only read the API spec, so they run without a target. - **Local Airflow runs without Docker by default.** Add `--docker` to `astro local start` for containers. uv 0.9.25 or later must be on your PATH. Windows needs `--docker`. -- **`astro local` needs a converted project.** It reads `pyproject.toml`, not `requirements.txt`, `packages.txt` or `airflow_settings.yaml`, so run `astro init` once in a 1.x project. `astro deploy` still deploys an unconverted 1.x project. +- **`astro local` needs a converted project.** It reads `pyproject.toml`, not `requirements.txt`, `packages.txt` or `airflow_settings.yaml`, so run `astro init` once in a 1.x project. `astro deploy` needs it converted too. - **Login tokens move into the OS keyring** (see above). A 1.x build cannot read them there, so it sees you as signed out and asks you to log in again. Once you do, v2 sees that 1.x uses that login and keeps it in `config.yaml` from then on, so both tools share it and neither signs the other out. `astro login --vault` logs in and moves the login back out of `config.yaml`, after which 1.x asks you to log in again. ## Installing v2 diff --git a/e2e/deployrefuse_test.go b/e2e/deployrefuse_test.go index 55563bb96..01417ad48 100644 --- a/e2e/deployrefuse_test.go +++ b/e2e/deployrefuse_test.go @@ -3,31 +3,31 @@ package e2e import ( + "os" "path/filepath" "strings" "testing" ) -// `astro deploy` in a project refuses the flags it would otherwise drop. +// `astro deploy` no longer has the flags only a 1.x project's deploy read, and +// a run passing one is told what replaced it. // -// The 1.x deploy accepted these and did something with them; the manifest path does -// not, and the failure worth preventing is the silent one — a CI job passing -// --pytest, watching the deploy succeed, and believing tests ran. Each refusal -// therefore names the flag and says what to do instead, and the text is the -// feature here rather than the exit code. +// The failure worth preventing is the silent one: a CI job passing --pytest, +// watching the deploy succeed, and believing tests ran. Each refusal therefore +// names the flag and says what to do instead, and the text is the feature +// here rather than the exit code. // -// Tier 0: nothing on this path is contacted. One scaffold serves every case, -// because refuseFlagsManifestDeployIgnores runs before the manifest is read and -// mutates nothing — each subtest takes a view bound to its own T so a failure -// stays in its own case. -func TestDeployRefusesTheFlagsAManifestProjectIgnores(t *testing.T) { +// Tier 0: the refusal happens while flags parse, before anything is read or +// contacted. One scaffold serves every case, and each subtest takes a view +// bound to its own T so a failure stays in its own case. +func TestDeployRefusesThe1xOnlyFlags(t *testing.T) { tier(t, 0) parent := manifestProjectForDeploy(t) - // The eight, written out rather than imported. Importing the table from - // cmd/astro would make this a test that the CLI agrees with itself; the - // point is that somebody decided these are the eight and said why. + // The eight, written out rather than imported. Importing the messages from + // cmd would make this a test that the CLI agrees with itself; the point + // is that somebody decided these are the eight and said why. for _, tc := range []struct { args []string // says is a fragment of the guidance, not the whole sentence: the @@ -36,12 +36,12 @@ func TestDeployRefusesTheFlagsAManifestProjectIgnores(t *testing.T) { }{ {[]string{"--pytest"}, "run your tests before deploying"}, {[]string{"--parse"}, "check your DAGs before deploying"}, - {[]string{"--dags-path", "somewhere"}, "deploy from the project directory"}, - {[]string{"--dag-bundle-name", "bundle"}, "named DAG bundles are not supported"}, + {[]string{"--dags-path", "somewhere"}, "ships the project's dags directory"}, + {[]string{"--dag-bundle-name", "bundle"}, "named DAG bundle is not supported yet"}, {[]string{"--test", "tests/"}, "run your tests before deploying"}, - {[]string{"--env", ".env.test"}, "this deploy runs no tests"}, - {[]string{"--save"}, "this deploy always asks"}, - {[]string{"--deployment-name", "prod"}, "name the target with --deployment"}, + {[]string{"--env", ".env.test"}, "runs no tests"}, + {[]string{"--save"}, "asks which Deployment to deploy to"}, + {[]string{"--deployment-name", "prod"}, "with --deployment"}, } { flag := strings.TrimPrefix(tc.args[0], "--") t.Run(flag, func(t *testing.T) { @@ -50,44 +50,158 @@ func TestDeployRefusesTheFlagsAManifestProjectIgnores(t *testing.T) { // Stderr, not the combined output: guidance a script is meant to // notice must not be on stdout, where --output json consumers - // parse. Reading the whole output would also pass on the usage - // block cobra prints underneath, which quotes several of these - // strings back in its own flag help. - if !strings.Contains(r.Stderr, "--"+flag+" has no effect") { - t.Errorf("the refusal should name the flag that was ignored\n%s", r.output()) + // parse. + if !strings.Contains(r.Stderr, "--"+flag+" was removed in Astro CLI v2") { + t.Errorf("the refusal should name the flag that was removed\n%s", r.output()) } if !strings.Contains(r.Stderr, tc.says) { t.Errorf("the refusal should say what to do instead (%q)\n%s", tc.says, r.output()) } - if !strings.Contains(r.Stderr, "a project with a pyproject.toml") { - t.Errorf("the refusal should say the pyproject.toml is why\n%s", r.output()) - } }) } - // And in json mode the refusal is an object, like every other failure on - // this path. It was not: refuseFlagsManifestDeployIgnores ran before the format - // was parsed, so `--output json` exited 1 with an empty stdout while the - // build-secret refusal below published {"error":...}. A script reading - // stdout to learn what went wrong got nothing — the same silence these - // refusals exist to break, one layer up. + // And in json mode the refusal is the error object, a usage error. t.Run("json mode publishes the refusal as an object", func(t *testing.T) { p := parent.forT(t) var payload struct { Error string `json:"error"` Code int `json:"code"` + Kind string `json:"kind"` } p.run("deploy", "--pytest", "--output", "json").requireFailure().requireJSON(&payload) - if !strings.Contains(payload.Error, "--pytest has no effect") { + if !strings.Contains(payload.Error, "--pytest was removed in Astro CLI v2") { t.Errorf("the object should carry the refusal, got %q", payload.Error) } - if payload.Code == 0 { - t.Error("a refusal should not publish code 0") + if payload.Code != 2 || payload.Kind != "usage" { + t.Errorf("want a usage error (code 2), got code %d kind %q", payload.Code, payload.Kind) } }) } +// make1xProject lays out a project the way Astro CLI 1.x made one: a +// Dockerfile, a .astro/config.yaml and dags/, and no pyproject.toml. +func make1xProject(t *testing.T, p *project) { + t.Helper() + write(t, filepath.Join(p.Dir, "Dockerfile"), "FROM quay.io/astronomer/astro-runtime:12.0.0\n") + mkdir(t, p.Dir, ".astro") + write(t, filepath.Join(p.Dir, ".astro", "config.yaml"), "project:\n name: legacy\n") + mkdir(t, p.Dir, "dags") +} + +// v2 deploys only pyproject.toml projects. A project in the 1.x layout is +// refused whatever the deploy asked for, with the two ways forward: convert +// it with astro init, or deploy it with Astro CLI 1.x. Under --output json it +// is the error object, of kind no_project. Nothing is contacted, and the +// project is left as it was: no file the deploy would have written (a +// .dockerignore edit, a saved Deployment) appears. +func TestDeployRefusesA1xProject(t *testing.T) { + tier(t, 0) + + parent := newProject(t) + make1xProject(t, parent) + before := listTree(t, parent.Dir) + + for _, args := range [][]string{ + {"deploy", "dep-id"}, + {"deploy", "dep-id", "--dags"}, + {"deploy", "dep-id", "--image"}, + } { + t.Run(strings.Join(args, " "), func(t *testing.T) { + p := parent.forT(t) + r := p.run(args...).requireFailure() + for _, want := range []string{"Astro CLI 1.x layout", "astro init", "deploy it with Astro CLI 1.x"} { + if !strings.Contains(r.Stderr, want) { + t.Errorf("the refusal should say %q\n%s", want, r.output()) + } + } + if strings.Contains(r.Stderr, "Usage:") { + t.Errorf("the refusal is not a usage mistake, so no usage block\n%s", r.output()) + } + + var payload struct { + Error string `json:"error"` + Kind string `json:"kind"` + } + p.run(append(args, "--output", "json")...).requireFailure().requireJSON(&payload) + if payload.Kind != "no_project" || !strings.Contains(payload.Error, "Astro CLI 1.x layout") { + t.Errorf("want the 1.x refusal of kind no_project, got %+v", payload) + } + }) + } + + if after := listTree(t, parent.Dir); strings.Join(after, "\n") != strings.Join(before, "\n") { + t.Errorf("a refused deploy changed the project\nbefore: %v\nafter: %v", before, after) + } +} + +// Outside any project the deploy gives the no-project advice, unchanged by +// the 1.x refusal: it says to run astro init, and says nothing of 1.x. +func TestDeployOutsideAProject(t *testing.T) { + tier(t, 0) + + p := newProject(t) + r := p.run("deploy", "dep-id").requireFailure() + if !strings.Contains(r.Stderr, "not an Astro project directory") || !strings.Contains(r.Stderr, "astro init") { + t.Errorf("want the no-project advice\n%s", r.output()) + } + if strings.Contains(r.Stderr, "1.x") { + t.Errorf("an empty directory is not a 1.x project\n%s", r.output()) + } +} + +// --image-name outside any project deploys an image already built, alone: it +// gets past the project check, to the container engine an image push needs. +// With Docker pointed at nothing (offline), that is where it stops, so +// nothing leaves the machine. In a 1.x project, or below a pyproject.toml +// project's root, it is refused as every deploy there is: a prebuilt image +// shipped from a 1.x checkout would leave that project's DAGs stale. +func TestDeployImageNameOutsideAProject(t *testing.T) { + tier(t, 0) + + t.Run("outside any project", func(t *testing.T) { + p := newProject(t) + r := runIn(t, p, "deploy", "dep-id", "--image-name", "img:1").requireFailure() + if !strings.Contains(r.Stderr, "an image deploy needs") { + t.Errorf("want it past the project check, stopped at the container engine\n%s", r.output()) + } + if strings.Contains(r.Stderr, "not an Astro project directory") { + t.Errorf("--image-name outside a project needs none\n%s", r.output()) + } + }) + + t.Run("in a 1.x project", func(t *testing.T) { + p := newProject(t) + make1xProject(t, p) + var payload struct { + Error string `json:"error"` + Kind string `json:"kind"` + } + runIn(t, p, "deploy", "dep-id", "--image-name", "img:1", "--force", "--output", "json").requireFailure().requireJSON(&payload) + if payload.Kind != "no_project" || !strings.Contains(payload.Error, "Astro CLI 1.x layout") { + t.Errorf("want the 1.x refusal of kind no_project, got %+v", payload) + } + }) +} + +// listTree is every path under dir, relative to it, in walk order. +func listTree(t *testing.T, dir string) []string { + t.Helper() + var paths []string + err := filepath.Walk(dir, func(path string, _ os.FileInfo, err error) error { + if err != nil { + return err + } + rel, err := filepath.Rel(dir, path) + paths = append(paths, rel) + return err + }) + if err != nil { + t.Fatalf("listing %s: %v", dir, err) + } + return paths +} + // `--build-secret` is the one that depends on the project, not just the flag. // // Without a declared Dockerfile only the netrc secret is accepted, because @@ -220,3 +334,55 @@ func TestDeployTakesANetrcBuildSecretWithoutADockerfile(t *testing.T) { t.Errorf("expected it to get past the build-secret gate and stop at the workspace\n%s", r.output()) } } + +// On Astro Private Cloud v2 builds no project yet: a deploy that would build +// one is refused, saying to use Astro CLI 1.x, before Houston is asked +// anything, and a project in the 1.x layout makes no deploy at all. The +// login is to a Houston that refuses every connection, so a refusal that +// came after a request would read as a connection error instead. +func TestAPCDeployRefusesToBuild(t *testing.T) { + tier(t, 0) + + for _, tc := range []struct { + name string + setup func(t *testing.T, p *project) + args []string + says []string + kind string + }{ + {"a 1.x project", make1xProject, []string{"deploy", "dep-id"}, []string{"Astro CLI 1.x layout", "Deploy it with Astro CLI 1.x"}, "no_project"}, + {"--image-name from a 1.x project", make1xProject, []string{"deploy", "dep-id", "--image-name", "img:1"}, []string{"Astro CLI 1.x layout"}, "no_project"}, + {"--dags from a 1.x project", make1xProject, []string{"deploy", "dep-id", "--dags"}, []string{"Astro CLI 1.x layout", "does not deploy to Astro Private Cloud"}, "no_project"}, + {"a pyproject.toml project", func(t *testing.T, p *project) { + p.forT(t).run("init", "--name", "deployable").requireSuccess() + }, []string{"deploy", "dep-id"}, []string{"cannot build and deploy projects to Astro Private Cloud yet", "use Astro CLI 1.x", "pyproject.toml projects on Astro Private Cloud is coming"}, "usage"}, + {"outside any project", func(*testing.T, *project) {}, []string{"deploy", "dep-id"}, []string{"cannot build and deploy projects to Astro Private Cloud yet", "--image-name"}, "no_project"}, + } { + t.Run(tc.name, func(t *testing.T) { + // With the keyring stamped unavailable, as the json walk's + // states are, so the login is read without asking the OS for it. + p := newStateProject(t, jsonState{}, "") + tc.setup(t, p) + writeLogin(t, p, "software") + + r := runIn(t, p, tc.args...).requireFailure() + for _, want := range tc.says { + if !strings.Contains(r.Stderr, want) { + t.Errorf("the refusal should say %q\n%s", want, r.output()) + } + } + if strings.Contains(r.output(), "connection refused") { + t.Errorf("the refusal should come before any request\n%s", r.output()) + } + + var payload struct { + Error string `json:"error"` + Kind string `json:"kind"` + } + runIn(t, p, append(tc.args, "--output", "json")...).requireFailure().requireJSON(&payload) + if payload.Kind != tc.kind || !strings.Contains(payload.Error, tc.says[0]) { + t.Errorf("want %q of kind %s, got %+v", tc.says[0], tc.kind, payload) + } + }) + } +} diff --git a/internal/deploy/deploy.go b/internal/deploy/deploy.go index 64573675b..836715c72 100644 --- a/internal/deploy/deploy.go +++ b/internal/deploy/deploy.go @@ -40,8 +40,8 @@ type Request struct { LinkName string // Deployment is --deployment. It names a manifest link, or — when the // project links nothing by that name — an Astro Deployment id, which is - // what the flag meant before it learned link names and what the 1.x path spells - // --deployment-id. Either way it names the target on the command line, + // what the flag meant before it learned link names and what Astro CLI 1.x + // spelled --deployment-id. Either way it names the target on the command line, // which is the only thing a deploy will resolve from. Deployment string // Preselect is what the ambient layers point at: ASTRO_DEPLOYMENT, then the @@ -61,6 +61,9 @@ type Request struct { DagsOnly bool // Image is --image: an image-only deploy, leaving the running dags in place. Image bool + // ImageAlone ships the image with no DAGs whatever the flags say: an + // --image-name deploy outside a project, which has no dags/ to take. + ImageAlone bool // ImageName is --image-name: a prebuilt local image to deploy instead of // building from the manifest. "" means build. ImageName string @@ -118,6 +121,9 @@ type Result struct { // LinkName is the manifest link the deploy resolved to, "" if unlinked. LinkName string Git Git + // DagDeployEnabled is whether the Deployment takes DAG deploys, as an + // image deploy read it; false for a dags-only deploy. + DagDeployEnabled bool } // Git is what a deploy recorded about the commit it shipped. @@ -192,6 +198,9 @@ type ImageResult struct { DagTarballVersion string URL string Git Git + // DagDeployEnabled is whether the Deployment takes DAG deploys, read + // while the image deploy checked it. + DagDeployEnabled bool } // Deployer is the seam onto the deploy transport. The CLI wires it to the @@ -232,7 +241,14 @@ func Run(req Request, d Deployer) (Result, error) { // with it. Checked before anything else, so an impossible combination fails // rather than asking a question whose answer it will throw away. if req.DagsOnly && (req.Image || req.ImageName != "") { - return Result{}, errors.New("--dags deploys only your DAGs; drop --image and --image-name") + var passed []string + if req.Image { + passed = append(passed, "--image") + } + if req.ImageName != "" { + passed = append(passed, "--image-name") + } + return Result{}, fmt.Errorf("--dags deploys only your DAGs; drop %s", strings.Join(passed, " and ")) } // Also before anything is asked: a project whose image source is refused // should not first make someone pick where to ship it. @@ -339,7 +355,7 @@ func runImage(req Request, target Target, d Deployer) (Result, error) { // == ""`, which turned an exported BUILD_SECRET_INPUT into a hard failure for // every project that generates its image. imagebuild passes a generated build // only the netrc secret, and checkImageSource has warned about the rest. - includeDags := !req.Image + includeDags := !req.Image && !req.ImageAlone img, err := d.DeployImage(&ImageDeploy{ DeploymentID: target.DeploymentID, WorkspaceID: target.WorkspaceID, @@ -370,6 +386,7 @@ func runImage(req Request, target Target, d Deployer) (Result, error) { URL: img.URL, LinkName: target.LinkName, Git: img.Git, + DagDeployEnabled: img.DagDeployEnabled, }, nil } @@ -409,11 +426,8 @@ func resolveTarget(req Request, d Deployer) (Target, error) { links = req.Manifest.Astro.Deployments } - if req.LinkName != "" && req.Deployment != "" && req.LinkName != req.Deployment { - return Target{}, fmt.Errorf("this deploy names two targets, %q and --deployment %q: name one", req.LinkName, req.Deployment) - } - if name := firstNonEmpty(req.LinkName, req.Deployment); name != "" { - return namedTarget(req, name, links) + if target, named, err := resolveNamed(req, links); named || err != nil { + return target, err } deployable := DeployableLinks(links) @@ -454,6 +468,32 @@ func resolveTarget(req Request, d Deployer) (Target, error) { return astroTarget(req, name, link, links) } +// ResolveNamed settles a target named on the command line, as Run does: the +// argument (linkName) and --deployment, which must agree, each a link of m +// (nil for no manifest) or a Deployment id. named is false when neither names +// one. It is for a command that ships something other than the project, such +// as astro deploy --non-dags, to name its Deployment the way a deploy does. +func ResolveNamed(m *manifest.Manifest, linkName, deployment, workspaceID, contextWorkspace string) (target Target, named bool, err error) { + req := Request{Manifest: m, LinkName: linkName, Deployment: deployment, WorkspaceID: workspaceID, ContextWorkspace: contextWorkspace} + var links map[string]manifest.Link + if m != nil { + links = m.Astro.Deployments + } + return resolveNamed(req, links) +} + +func resolveNamed(req Request, links map[string]manifest.Link) (Target, bool, error) { + if req.LinkName != "" && req.Deployment != "" && req.LinkName != req.Deployment { + return Target{}, true, fmt.Errorf("this deploy names two targets, %q and --deployment %q: name one", req.LinkName, req.Deployment) + } + name := firstNonEmpty(req.LinkName, req.Deployment) + if name == "" { + return Target{}, false, nil + } + target, err := namedTarget(req, name, links) + return target, true, err +} + // namedTarget resolves a target named on the command line, by the positional // argument or --deployment alike: a manifest link, else an Astro Deployment id. // The id fall-through is what both meant before they learned link names — the @@ -477,6 +517,10 @@ func unlinkedTarget(req Request, d Deployer) (Target, error) { return Target{}, errors.New("a workspace is required for a deploy: pass --workspace or set a current workspace") } if !req.Interactive { + if req.Manifest == nil { + // No project: there is no link to name, only an id. + return Target{}, input.Required(errors.New("this deploy names no Deployment and this run cannot be asked: pass the Deployment id as the argument or with --deployment")) + } return Target{}, input.Required(errors.New("this project links no deployment and this run cannot be asked: pass --deployment ")) } id, err := d.ResolveUnlinked(workspaceID) diff --git a/internal/platform/apc/deploy/custom_image_test.go b/internal/platform/apc/deploy/custom_image_test.go index 1ff7d496f..eb4286684 100644 --- a/internal/platform/apc/deploy/custom_image_test.go +++ b/internal/platform/apc/deploy/custom_image_test.go @@ -39,7 +39,7 @@ func (s *Suite) TestCustomImageToTheAPCRegistryTakesTheNextTag() { return s.mockImageHandler } - got, err := buildPushDockerImage(s.houstonMock, &config.Context{}, mockDeployment, "my-release", "./testfiles/", c.nextTag, "example.com", "", false, false, description, c.image, Options{}) + got, err := tagPushDockerImage(s.houstonMock, &config.Context{}, mockDeployment, "my-release", c.nextTag, "example.com", "", false, c.image, Options{}) s.NoError(err, c.image) s.Equal(c.want, pushedTo, c.image) s.Equal(c.want, got) diff --git a/internal/platform/apc/deploy/deploy.go b/internal/platform/apc/deploy/deploy.go index 5927db20d..cea71029f 100644 --- a/internal/platform/apc/deploy/deploy.go +++ b/internal/platform/apc/deploy/deploy.go @@ -16,9 +16,7 @@ import ( "golang.org/x/mod/semver" "github.com/astronomer/astro-cli/airflow" - "github.com/astronomer/astro-cli/airflow/types" "github.com/astronomer/astro-cli/config" - "github.com/astronomer/astro-cli/docker" "github.com/astronomer/astro-cli/internal/platform/apc/auth" "github.com/astronomer/astro-cli/internal/platform/apc/houston" "github.com/astronomer/astro-cli/pkg/fileutil" @@ -31,10 +29,6 @@ var ( // this is used to monkey patch the function in order to write unit test cases imageHandlerInit = airflow.ImageHandlerInit - dockerfile = "Dockerfile" - - deployImagePlatformSupport = []string{"linux/amd64"} - gzipFile = fileutil.GzipFile getDeploymentIDForCurrentCommandVar = getDeploymentIDForCurrentCommand @@ -70,7 +64,6 @@ var ( // under astronomer.houston.config in the platform's values. ErrBYORegistryDomainNotSet = errors.New("Custom registry host is not set in config. It can be set at astronomer.houston.config.registry.protectedCustomRegistry.updateRegistry.host") ErrDeploymentTypeIncorrectForImageOnly = errors.New("--image only works for Dag-only, Git-sync-based and NFS-based deployments") - WarningInvalidImageNameMsg = "WARNING! The image in your Dockerfile '%s' is not based on Astro Runtime and is not supported. Change your Dockerfile with an image that pulls from 'quay.io/astronomer/astro-runtime' to proceed.\n" ErrNoRuntimeLabelOnCustomImage = errors.New("the image should have label io.astronomer.docker.runtime.version") ErrRuntimeVersionNotPassedForRemoteImage = errors.New("if --image-name and --remote is passed, it's mandatory to pass --runtime-version") ) @@ -80,9 +73,6 @@ const ( houstonSelectDeploymentPrompt = "Select which airflow deployment you want to deploy to:" houstonDeploymentPrompt = "Deploying: %s\n" - imageBuildingPrompt = "Building image..." - - warningInvalidImageName = "WARNING! The image in your Dockerfile is pulling from '%s', which is not supported. We strongly recommend that you use Astronomer Certified or Runtime images that pull from 'astronomerinc/ap-airflow', 'quay.io/astronomer/ap-airflow' or 'quay.io/astronomer/astro-runtime'. If you're running a custom image, you can override this. Are you sure you want to continue?\n" warningInvalidNameTag = "WARNING! You are about to push an image using the '%s' tag. This is not recommended.\nPlease use one of the following tags: %s.\nAre you sure you want to continue?" warningInvalidNameTagEmptyRecommendations = "WARNING! You are about to push an image using the '%s' tag. This is not recommended.\nAre you sure you want to continue?" @@ -186,10 +176,11 @@ func (o Options) progress() io.Writer { // other than stdout (airflow.DockerImage). type progressSetter interface{ ProgressTo(io.Writer) } -// Airflow builds the project's image, or tags the one named, pushes it to -// the Deployment's registry and returns what it pushed. Its DeploymentID is -// set on a failure too, once the Deployment is known. -func Airflow(houstonClient houston.ClientInterface, path, deploymentID, wsID string, ignoreCacheDeploy, prompt bool, description string, isImageOnlyDeploy bool, imageName string, opts Options) (Deployed, error) { +// Airflow tags the local image named (--image-name), pushes it to the +// Deployment's registry and returns what it pushed. It builds nothing: v2 +// does not build projects for APC. Its DeploymentID is set on a failure too, +// once the Deployment is known. +func Airflow(houstonClient houston.ClientInterface, deploymentID, wsID string, prompt, isImageOnlyDeploy bool, imageName string, opts Options) (Deployed, error) { deploymentID, deployments, err := getDeploymentIDForCurrentCommand(houstonClient, wsID, deploymentID, prompt) if err != nil { return Deployed{DeploymentID: deploymentID}, err @@ -240,8 +231,7 @@ func Airflow(houstonClient houston.ClientInterface, path, deploymentID, wsID str fmt.Fprintf(opts.progress(), houstonDeploymentPrompt, releaseName) - // Build the image to deploy - pushed, err := buildPushDockerImage(houstonClient, &c, deploymentInfo, releaseName, path, nextTag, cloudDomain, byoRegistryDomain, ignoreCacheDeploy, byoRegistryEnabled, description, imageName, opts) + pushed, err := tagPushDockerImage(houstonClient, &c, deploymentInfo, releaseName, nextTag, cloudDomain, byoRegistryDomain, byoRegistryEnabled, imageName, opts) if err != nil { return Deployed{DeploymentID: deploymentID}, err } @@ -420,7 +410,9 @@ func pushDockerImage(byoRegistryEnabled bool, deploymentInfo *houston.Deployment return remoteImage, nil } -func buildDockerImageForCustomImage(imageHandler airflow.ImageHandler, customImageName string, deploymentInfo *houston.Deployment, houstonClient houston.ClientInterface, opts Options) error { +// tagCustomImage tags the local image customImageName as the Deployment's +// and checks the runtime its label names against the platform's. +func tagCustomImage(imageHandler airflow.ImageHandler, customImageName string, deploymentInfo *houston.Deployment, houstonClient houston.ClientInterface, opts Options) error { fmt.Fprintln(opts.progress(), composeSkipImageBuildingPromptMsg) err := imageHandler.TagLocalImage(customImageName) if err != nil { @@ -438,45 +430,6 @@ func buildDockerImageForCustomImage(imageHandler airflow.ImageHandler, customIma return err } -func buildDockerImageFromWorkingDir(path string, imageHandler airflow.ImageHandler, houstonClient houston.ClientInterface, deploymentInfo *houston.Deployment, ignoreCacheDeploy bool, description string, opts Options) error { - // all these checks inside Dockerfile should happen only when no image-name is provided - // parse dockerfile - cmds, err := docker.ParseFile(filepath.Join(path, dockerfile)) - if err != nil { - return fmt.Errorf("failed to parse dockerfile: %s: %w", filepath.Join(path, dockerfile), err) - } - - _, tag := docker.GetImageTagFromParsedFile(cmds) - - // Get valid image tags for platform using Deployment Info request - err = validateRuntimeVersion(houstonClient, tag, deploymentInfo, opts) - if err != nil { - return err - } - // Build our image - fmt.Fprintln(opts.progress(), imageBuildingPrompt) - deployLabels := []string{"io.astronomer.skip.revision=true"} - if description != "" { - deployLabels = append(deployLabels, "io.astronomer.deploy.revision.description="+description) - } - buildConfig := types.ImageBuildConfig{ - Path: config.WorkingPath, - NoCache: ignoreCacheDeploy, - TargetPlatforms: deployImagePlatformSupport, - Labels: deployLabels, - } - - err = imageHandler.Build("", nil, buildConfig) - return err -} - -func buildDockerImage(ignoreCacheDeploy bool, deploymentInfo *houston.Deployment, customImageName, path string, imageHandler airflow.ImageHandler, houstonClient houston.ClientInterface, description string, opts Options) error { - if customImageName == "" { - return buildDockerImageFromWorkingDir(path, imageHandler, houstonClient, deploymentInfo, ignoreCacheDeploy, description, opts) - } - return buildDockerImageForCustomImage(imageHandler, customImageName, deploymentInfo, houstonClient, opts) -} - func getGetTagFromImageName(imageName string) string { parts := strings.Split(imageName, ":") if len(parts) == 2 { @@ -485,13 +438,13 @@ func getGetTagFromImageName(imageName string) string { return "" } -func buildPushDockerImage(houstonClient houston.ClientInterface, c *config.Context, deploymentInfo *houston.Deployment, name, path, nextTag, cloudDomain, byoRegistryDomain string, ignoreCacheDeploy, byoRegistryEnabled bool, description, customImageName string, opts Options) (string, error) { +func tagPushDockerImage(houstonClient houston.ClientInterface, c *config.Context, deploymentInfo *houston.Deployment, name, nextTag, cloudDomain, byoRegistryDomain string, byoRegistryEnabled bool, customImageName string, opts Options) (string, error) { imageName := airflow.ImageName(name, "latest") imageHandler := imageHandlerInit(imageName) if p, ok := imageHandler.(progressSetter); ok { p.ProgressTo(opts.progress()) } - err := buildDockerImage(ignoreCacheDeploy, deploymentInfo, customImageName, path, imageHandler, houstonClient, description, opts) + err := tagCustomImage(imageHandler, customImageName, deploymentInfo, houstonClient, opts) if err != nil { return "", err } @@ -550,11 +503,6 @@ func getDeploymentIDForCurrentCommand(houstonClient houston.ClientInterface, wsI return deploymentID, deployments, errNoDomainSet } - // Use config deployment if provided - if deploymentID == "" { - deploymentID = config.CFG.ProjectDeployment.GetProjectString() - } - if deploymentID != "" && !deploymentExists(deploymentID, deployments) { return deploymentID, deployments, errInvalidDeploymentID } diff --git a/internal/platform/apc/deploy/deploy_test.go b/internal/platform/apc/deploy/deploy_test.go index 22675b370..2450c39ed 100644 --- a/internal/platform/apc/deploy/deploy_test.go +++ b/internal/platform/apc/deploy/deploy_test.go @@ -20,7 +20,6 @@ import ( "github.com/astronomer/astro-cli/airflow" "github.com/astronomer/astro-cli/airflow/mocks" - "github.com/astronomer/astro-cli/airflow/types" "github.com/astronomer/astro-cli/config" "github.com/astronomer/astro-cli/context" "github.com/astronomer/astro-cli/internal/platform/apc/houston" @@ -31,10 +30,9 @@ import ( ) var ( - errSomeContainerIssue = errors.New("some container issue") - errMockHouston = errors.New("some houston error") - description = "Deployed via " - deployRevisionDescriptionLabel = "io.astronomer.deploy.revision.description" + errSomeContainerIssue = errors.New("some container issue") + errMockHouston = errors.New("some houston error") + description = "Deployed via " mockDeployment = &houston.Deployment{ ID: "cknz133ra49758zr9w34b87ua", @@ -144,139 +142,38 @@ func (s *Suite) TearDownTest() { s.mockImageHandler.AssertExpectations(s.T()) } -func (s *Suite) TestBuildPushDockerImageSuccessWithTagWarning() { +func (s *Suite) TestTagPushDockerImageSuccessWithBYORegistry() { config.InitConfig(s.fsForDockerConfig) - context.Switch("localhost") - dockerfile = "Dockerfile.warning" - defer func() { dockerfile = "Dockerfile" }() - - imageHandlerInit = func(image string) airflow.ImageHandler { - s.mockImageHandler.On("Build", mock.Anything, mock.Anything, mock.Anything).Return(nil) - s.mockImageHandler.On("Push", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return("", nil) - return s.mockImageHandler - } - - defer testUtil.MockUserInput(s.T(), "y")() - - mockedDeploymentConfig := &houston.DeploymentConfig{ - AirflowImages: mockAirflowImageList, - } - vars := make(map[string]interface{}) - vars["clusterId"] = "" - s.houstonMock.On("GetRuntimeReleases", vars).Return(houston.RuntimeReleases{}, nil) - s.houstonMock.On("GetDeploymentConfig", nil).Return(mockedDeploymentConfig, nil) - s.houstonMock.On("GetPlatformVersion", mock.Anything).Return("1.0.0", nil).Once() - - _, err := buildPushDockerImage(s.houstonMock, &config.Context{}, mockDeployment, "test", "./testfiles/", "test", "test", "", false, false, description, "", Options{}) - s.NoError(err) -} - -func (s *Suite) TestBuildPushDockerImageSuccessWithImageRepoWarning() { - config.InitConfig(s.fsForDockerConfig) - context.Switch("localhost") - dockerfile = "Dockerfile.privateImageRepo" - defer func() { dockerfile = "Dockerfile" }() - - imageHandlerInit = func(image string) airflow.ImageHandler { - s.mockImageHandler.On("Build", mock.Anything, mock.Anything, mock.Anything).Return(nil) - s.mockImageHandler.On("Push", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return("", nil) - return s.mockImageHandler - } - - defer testUtil.MockUserInput(s.T(), "y")() - - mockedDeploymentConfig := &houston.DeploymentConfig{ - AirflowImages: mockAirflowImageList, - } - vars := make(map[string]interface{}) - vars["clusterId"] = "" - s.houstonMock.On("GetDeploymentConfig", nil).Return(mockedDeploymentConfig, nil) - s.houstonMock.On("GetRuntimeReleases", vars).Return(houston.RuntimeReleases{}, nil) - s.houstonMock.On("GetPlatformVersion", mock.Anything).Return("1.0.0", nil).Once() - - _, err := buildPushDockerImage(s.houstonMock, &config.Context{}, mockDeployment, "test", "./testfiles/", "test", "test", "", false, false, description, "", Options{}) - s.NoError(err) -} - -func (s *Suite) TestBuildPushDockerImageSuccessWithBYORegistry() { - config.InitConfig(s.fsForDockerConfig) - dockerfile = "Dockerfile" - defer func() { dockerfile = "Dockerfile" }() - - var capturedBuildConfig types.ImageBuildConfig + customImageName := "test-image-name:v1" imageHandlerInit = func(image string) airflow.ImageHandler { - // Mock the Build function, capturing the buildConfig - s.mockImageHandler.On("Build", mock.Anything, mock.Anything, mock.MatchedBy(func(buildConfig types.ImageBuildConfig) bool { - // Capture buildConfig for later assertions - capturedBuildConfig = buildConfig - // Check if the deploy label contains the correct description - for _, label := range buildConfig.Labels { - if label == deployRevisionDescriptionLabel+"="+description { - return true - } - } - return false - })).Return(nil).Once() - - s.mockImageHandler.On("Push", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return("image_sha", nil) - s.mockImageHandler.On("GetLabel", "", runtimeImageLabel).Return("", nil).Once() + s.mockImageHandler.On("TagLocalImage", customImageName).Return(nil).Once() + s.mockImageHandler.On("Push", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return("image_sha", nil).Once() + s.mockImageHandler.On("GetLabel", "", runtimeImageLabel).Return("12.2.0", nil).Twice() s.mockImageHandler.On("GetLabel", "", airflowImageLabel).Return("1.10.12", nil).Once() - return s.mockImageHandler } - mockedDeploymentConfig := &houston.DeploymentConfig{ - AirflowImages: mockAirflowImageList, - } - s.houstonMock.On("GetDeploymentConfig", nil).Return(mockedDeploymentConfig, nil) + s.houstonMock.On("GetDeploymentConfig", nil).Return(&houston.DeploymentConfig{AirflowImages: mockAirflowImageList}, nil) vars := make(map[string]interface{}) vars["clusterId"] = "" s.houstonMock.On("GetRuntimeReleases", vars).Return(houston.RuntimeReleases{}, nil) - s.houstonMock.On("UpdateDeploymentImage", houston.UpdateDeploymentImageRequest{ReleaseName: "test", Image: "test.registry.io:test-test", AirflowVersion: "1.10.12", RuntimeVersion: ""}).Return(nil, nil) + s.houstonMock.On("UpdateDeploymentImage", houston.UpdateDeploymentImageRequest{ReleaseName: "test", Image: "test.registry.io:v1", AirflowVersion: "1.10.12", RuntimeVersion: "12.2.0"}).Return(nil, nil).Once() - _, err := buildPushDockerImage(s.houstonMock, &config.Context{}, mockDeployment, "test", "./testfiles/", "test", "test", "test.registry.io", false, true, description, "", Options{}) + _, err := tagPushDockerImage(s.houstonMock, &config.Context{}, mockDeployment, "test", "test", "test", "test.registry.io", true, customImageName, Options{}) s.NoError(err) - expectedLabel := deployRevisionDescriptionLabel + "=" + description - assert.Contains(s.T(), capturedBuildConfig.Labels, expectedLabel) - - // Set up expectations for SHA tag test - s.houstonMock.On("UpdateDeploymentImage", houston.UpdateDeploymentImageRequest{ReleaseName: "test", Image: "test.registry.io@image_sha", AirflowVersion: "1.10.12", RuntimeVersion: ""}).Return(nil, nil) - - // Reset image handler for SHA tag test + // With sha_as_tag, the BYO registry takes the image by its digest. + s.houstonMock.On("UpdateDeploymentImage", houston.UpdateDeploymentImageRequest{ReleaseName: "test", Image: "test.registry.io@image_sha", AirflowVersion: "1.10.12", RuntimeVersion: "12.2.0"}).Return(nil, nil).Once() s.mockImageHandler = new(mocks.ImageHandler) - imageHandlerInit = func(image string) airflow.ImageHandler { - // Mock the Build function, capturing the buildConfig - s.mockImageHandler.On("Build", mock.Anything, mock.Anything, mock.MatchedBy(func(buildConfig types.ImageBuildConfig) bool { - // Capture buildConfig for later assertions - capturedBuildConfig = buildConfig - // Check if the deploy label contains the correct description - for _, label := range buildConfig.Labels { - if label == deployRevisionDescriptionLabel+"="+description { - return true - } - } - return false - })).Return(nil).Once() - - s.mockImageHandler.On("Push", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return("image_sha", nil) - s.mockImageHandler.On("GetLabel", "", runtimeImageLabel).Return("", nil).Once() - s.mockImageHandler.On("GetLabel", "", airflowImageLabel).Return("1.10.12", nil).Once() - return s.mockImageHandler - } config.CFG.ShaAsTag.SetHomeString("true") defer config.CFG.ShaAsTag.SetHomeString("false") - _, err = buildPushDockerImage(s.houstonMock, &config.Context{}, mockDeployment, "test", "./testfiles/", "test", "test", "test.registry.io", false, true, description, "", Options{}) + _, err = tagPushDockerImage(s.houstonMock, &config.Context{}, mockDeployment, "test", "test", "test", "test.registry.io", true, customImageName, Options{}) s.NoError(err) - expectedLabel = deployRevisionDescriptionLabel + "=" + description - assert.Contains(s.T(), capturedBuildConfig.Labels, expectedLabel) } -func (s *Suite) TestBuildPushDockerImageSuccessWithBYORegistryAndCustomImageName() { +func (s *Suite) TestTagPushDockerImageSuccessWithBYORegistryAndCustomImageName() { config.InitConfig(s.fsForDockerConfig) - dockerfile = "Dockerfile" - defer func() { dockerfile = "Dockerfile" }() customImageName := "test-image-name:latest" imageHandlerInit = func(image string) airflow.ImageHandler { @@ -297,53 +194,71 @@ func (s *Suite) TestBuildPushDockerImageSuccessWithBYORegistryAndCustomImageName s.houstonMock.On("GetRuntimeReleases", vars).Return(houston.RuntimeReleases{}, nil) s.houstonMock.On("UpdateDeploymentImage", houston.UpdateDeploymentImageRequest{ReleaseName: "test", Image: "test.registry.io:latest", AirflowVersion: "1.10.12", RuntimeVersion: "12.2.0"}).Return(nil, nil) - _, err := buildPushDockerImage(s.houstonMock, &config.Context{}, mockDeployment, "test", "./testfiles/", "test", "test", "test.registry.io", false, true, description, customImageName, Options{}) + _, err := tagPushDockerImage(s.houstonMock, &config.Context{}, mockDeployment, "test", "test", "test", "test.registry.io", true, customImageName, Options{}) s.NoError(err) } -func (s *Suite) TestBuildPushDockerImageFailure() { - // invalid dockerfile test - dockerfile = "Dockerfile.invalid" - _, err := buildPushDockerImage(nil, &config.Context{}, mockDeployment, "test", "./testfiles/", "test", "test", "", false, false, description, "", Options{}) - s.EqualError(err, "failed to parse dockerfile: testfiles/Dockerfile.invalid: when using JSON array syntax, arrays must be comprised of strings only") - dockerfile = "Dockerfile" - +func (s *Suite) TestTagPushDockerImageFailure() { config.InitConfig(s.fsForDockerConfig) context.Switch("localhost") - mockedDeploymentConfig := &houston.DeploymentConfig{ - AirflowImages: mockAirflowImageList, - } - s.houstonMock.On("GetDeploymentConfig", nil).Return(nil, errMockHouston).Once() - vars := make(map[string]interface{}) - vars["clusterId"] = "" - s.houstonMock.On("GetRuntimeReleases", vars).Return(houston.RuntimeReleases{}, nil) - // houston GetDeploymentConfig call failure - _, err = buildPushDockerImage(s.houstonMock, &config.Context{}, mockDeployment, "test", "./testfiles/", "test", "test", "", false, false, description, "", Options{}) - s.Error(err, errMockHouston) - - s.houstonMock.On("GetDeploymentConfig", nil).Return(mockedDeploymentConfig, nil).Twice() + customImageName := "test-image-name:v1" + // The image cannot be tagged. imageHandlerInit = func(image string) airflow.ImageHandler { - s.mockImageHandler.On("Build", mock.Anything, mock.Anything, mock.Anything).Return(errSomeContainerIssue) + s.mockImageHandler.On("TagLocalImage", customImageName).Return(errSomeContainerIssue).Once() return s.mockImageHandler } + _, err := tagPushDockerImage(s.houstonMock, &config.Context{}, mockDeployment, "test", "test", "test", "", false, customImageName, Options{}) + s.ErrorIs(err, errSomeContainerIssue) - // build error test case - _, err = buildPushDockerImage(s.houstonMock, &config.Context{}, mockDeployment, "test", "./testfiles/", "test", "test", "", false, false, description, "", Options{}) - s.Error(err, errSomeContainerIssue.Error()) - s.mockImageHandler.AssertExpectations(s.T()) - + // houston GetDeploymentConfig call failure s.mockImageHandler = new(mocks.ImageHandler) imageHandlerInit = func(image string) airflow.ImageHandler { - s.mockImageHandler.On("Build", mock.Anything, mock.Anything, mock.Anything).Return(nil) - s.mockImageHandler.On("Push", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return("", errSomeContainerIssue) + s.mockImageHandler.On("TagLocalImage", customImageName).Return(nil).Once() + s.mockImageHandler.On("GetLabel", "", runtimeImageLabel).Return("12.2.0", nil).Once() return s.mockImageHandler } - s.houstonMock.On("GetPlatformVersion", mock.Anything).Return("1.0.0", nil).Once() + s.houstonMock.On("GetDeploymentConfig", nil).Return(nil, errMockHouston).Once() + _, err = tagPushDockerImage(s.houstonMock, &config.Context{}, mockDeployment, "test", "test", "test", "", false, customImageName, Options{}) + s.ErrorIs(err, errMockHouston) // push error test case - _, err = buildPushDockerImage(s.houstonMock, &config.Context{}, mockDeployment, "test", "./testfiles/", "test", "test", "", false, false, description, "", Options{}) - s.Error(err, errSomeContainerIssue.Error()) + s.mockImageHandler = new(mocks.ImageHandler) + imageHandlerInit = func(image string) airflow.ImageHandler { + s.mockImageHandler.On("TagLocalImage", customImageName).Return(nil).Once() + s.mockImageHandler.On("GetLabel", "", runtimeImageLabel).Return("12.2.0", nil).Once() + s.mockImageHandler.On("Push", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return("", errSomeContainerIssue).Once() + return s.mockImageHandler + } + s.houstonMock.On("GetDeploymentConfig", nil).Return(&houston.DeploymentConfig{AirflowImages: mockAirflowImageList}, nil).Once() + vars := make(map[string]interface{}) + vars["clusterId"] = "" + s.houstonMock.On("GetRuntimeReleases", vars).Return(houston.RuntimeReleases{}, nil).Once() + s.houstonMock.On("GetPlatformVersion", mock.Anything).Return("0.30.0", nil).Once() + _, err = tagPushDockerImage(s.houstonMock, &config.Context{}, mockDeployment, "test", "test", "test", "", false, customImageName, Options{}) + s.ErrorIs(err, errSomeContainerIssue) +} + +// A deploy that names no Deployment asks which one. The project.deployment a +// 1.x project's .astro/config.yaml saved (astro deploy --save) is not used: +// v2 deploys no 1.x project, and a stale one would ship somewhere unasked. +func (s *Suite) TestDeploymentIDIgnoresTheSaved1xDeployment() { + fs := afero.NewMemMapFs() + s.Require().NoError(afero.WriteFile(fs, config.HomeConfigFile, testUtil.NewTestConfig(testUtil.SoftwarePlatform), 0o600)) + dir := s.T().TempDir() + prev := config.WorkingPath + config.WorkingPath = dir + s.T().Cleanup(func() { config.WorkingPath = prev; testUtil.InitTestConfig(testUtil.SoftwarePlatform) }) + s.Require().NoError(afero.WriteFile(fs, filepath.Join(dir, config.ConfigDir, config.ConfigFileNameWithExt), []byte("project:\n deployment: stale-id\n"), 0o600)) + config.InitConfig(fs) + + s.houstonMock.On("GetWorkspace", "ws").Return(&houston.Workspace{ID: "ws"}, nil).Once() + s.houstonMock.On("ListDeployments", mock.Anything).Return([]houston.Deployment{{ID: "stale-id"}, {ID: "picked"}}, nil).Once() + defer testUtil.MockUserInput(s.T(), "2\n")() + + got, _, err := getDeploymentIDForCurrentCommand(s.houstonMock, "ws", "", false) + s.NoError(err) + s.Equal("picked", got, "the picker decides, not the saved deployment") } func (s *Suite) TestGetAirflowUILink() { @@ -435,13 +350,13 @@ func (s *Suite) TestGetDagDeployURL() { func (s *Suite) TestAirflowFailure() { // No workspace ID test case - _, err := Airflow(nil, "", "", "", false, false, description, false, "", Options{}) + _, err := Airflow(nil, "", "", false, false, "img:1", Options{}) s.ErrorIs(err, ErrNoWorkspaceID) // houston GetWorkspace failure case s.houstonMock.On("GetWorkspace", mock.Anything).Return(nil, errMockHouston).Once() - _, err = Airflow(s.houstonMock, "", "", "test-workspace-id", false, false, description, false, "", Options{}) + _, err = Airflow(s.houstonMock, "", "test-workspace-id", false, false, "img:1", Options{}) s.ErrorIs(err, errMockHouston) s.houstonMock.AssertExpectations(s.T()) @@ -449,7 +364,7 @@ func (s *Suite) TestAirflowFailure() { s.houstonMock.On("GetWorkspace", mock.Anything).Return(&houston.Workspace{}, nil) s.houstonMock.On("ListDeployments", mock.Anything).Return(nil, errMockHouston).Once() - _, err = Airflow(s.houstonMock, "", "", "test-workspace-id", false, false, description, false, "", Options{}) + _, err = Airflow(s.houstonMock, "", "test-workspace-id", false, false, "img:1", Options{}) s.ErrorIs(err, errMockHouston) s.houstonMock.AssertExpectations(s.T()) @@ -460,46 +375,48 @@ func (s *Suite) TestAirflowFailure() { // config GetCurrentContext failure case config.ResetCurrentContext() - _, err = Airflow(s.houstonMock, "", "", "test-workspace-id", false, false, description, false, "", Options{}) + _, err = Airflow(s.houstonMock, "", "test-workspace-id", false, false, "img:1", Options{}) s.EqualError(err, "no context set, have you authenticated to Astro or APC? Run astro login and try again") context.Switch("localhost") // Invalid deployment name case - _, err = Airflow(s.houstonMock, "", "test-deployment-id", "test-workspace-id", false, false, description, false, "", Options{}) + _, err = Airflow(s.houstonMock, "test-deployment-id", "test-workspace-id", false, false, "img:1", Options{}) s.ErrorIs(err, errInvalidDeploymentID) // No deployment in the current workspace case - _, err = Airflow(s.houstonMock, "", "", "test-workspace-id", false, false, description, false, "", Options{}) + _, err = Airflow(s.houstonMock, "", "test-workspace-id", false, false, "img:1", Options{}) s.ErrorIs(err, errDeploymentNotFound) s.houstonMock.AssertExpectations(s.T()) // Invalid deployment selection case s.houstonMock.On("ListDeployments", mock.Anything).Return([]houston.Deployment{{ID: "test-deployment-id"}}, nil) - _, err = Airflow(s.houstonMock, "", "", "test-workspace-id", false, false, description, false, "", Options{}) + _, err = Airflow(s.houstonMock, "", "test-workspace-id", false, false, "img:1", Options{}) s.ErrorIs(err, errInvalidDeploymentSelected) // return error When houston get deployment throws an error s.houstonMock.On("ListDeployments", mock.Anything).Return([]houston.Deployment{{ID: "test-deployment-id"}}, nil) s.houstonMock.On("GetDeployment", mock.Anything).Return(nil, errMockHouston).Once() - _, err = Airflow(s.houstonMock, "", "test-deployment-id", "test-workspace-id", false, false, description, false, "", Options{}) + _, err = Airflow(s.houstonMock, "test-deployment-id", "test-workspace-id", false, false, "img:1", Options{}) s.Equal(err.Error(), "failed to get deployment info: "+errMockHouston.Error()) - // buildPushDockerImage failure case + // tagPushDockerImage failure case s.houstonMock.On("GetDeployment", "test-deployment-id").Return(&houston.Deployment{ClusterID: "test-cluster-id"}, nil) s.houstonMock.On("GetAppConfig", mock.Anything).Return(&houston.AppConfig{}, nil) - dockerfile = "Dockerfile.invalid" - _, err = Airflow(s.houstonMock, "./testfiles/", "test-deployment-id", "test-workspace-id", false, false, description, false, "", Options{}) - dockerfile = "Dockerfile" - s.Error(err) - s.Contains(err.Error(), "failed to parse dockerfile") + imageHandlerInit = func(image string) airflow.ImageHandler { + s.mockImageHandler.On("TagLocalImage", "img:1").Return(errSomeContainerIssue).Once() + return s.mockImageHandler + } + _, err = Airflow(s.houstonMock, "test-deployment-id", "test-workspace-id", false, false, "img:1", Options{}) + s.ErrorIs(err, errSomeContainerIssue) } func (s *Suite) TestAirflowSuccess() { config.InitConfig(s.fsForLocalConfig) imageHandlerInit = func(image string) airflow.ImageHandler { - s.mockImageHandler.On("Build", mock.Anything, mock.Anything, mock.Anything).Return(nil).Once() + s.mockImageHandler.On("TagLocalImage", "img:1").Return(nil).Once() + s.mockImageHandler.On("GetLabel", "", airflow.RuntimeImageLabel).Return("4.2.5", nil) s.mockImageHandler.On("Push", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return("", nil).Once() return s.mockImageHandler } @@ -533,7 +450,7 @@ func (s *Suite) TestAirflowSuccess() { var deployed Deployed var err error printed := stdoutOf(s, func() { - deployed, err = Airflow(s.houstonMock, "./testfiles/", "test-deployment-id", "test-workspace-id", false, false, description, false, "", Options{Progress: progress}) + deployed, err = Airflow(s.houstonMock, "test-deployment-id", "test-workspace-id", false, false, "img:1", Options{Progress: progress}) }) s.NoError(err) s.Contains(progress.String(), "Pushing image to configured registry") @@ -551,7 +468,8 @@ func (s *Suite) TestAirflowSuccessForBYORegistry() { config.InitConfig(s.fsForLocalConfig) imageHandlerInit = func(image string) airflow.ImageHandler { - s.mockImageHandler.On("Build", mock.Anything, mock.Anything, mock.Anything).Return(nil).Once() + s.mockImageHandler.On("TagLocalImage", "img:1").Return(nil).Once() + s.mockImageHandler.On("GetLabel", "", airflow.RuntimeImageLabel).Return("4.2.5", nil) s.mockImageHandler.On("Push", mock.MatchedBy(func(remoteImage string) bool { return strings.Contains(remoteImage, "my.registry.domain") }), mock.Anything, mock.Anything, mock.Anything).Return("", nil).Once() s.mockImageHandler.On("GetLabel", "", airflow.RuntimeImageLabel).Return("4.2.5", nil) s.mockImageHandler.On("GetLabel", "", airflowImageLabel).Return("2.2.5", nil) @@ -587,7 +505,7 @@ func (s *Suite) TestAirflowSuccessForBYORegistry() { s.houstonMock.On("GetRuntimeReleases", vars).Return(mockRuntimeReleases, nil) s.houstonMock.On("UpdateDeploymentImage", mock.Anything).Return(&houston.UpdateDeploymentImageResp{}, nil).Once() - deployed, err := Airflow(s.houstonMock, "./testfiles/", "test-deployment-id", "test-workspace-id", false, false, description, false, "", Options{}) + deployed, err := Airflow(s.houstonMock, "test-deployment-id", "test-workspace-id", false, false, "img:1", Options{}) s.NoError(err) s.True(strings.HasPrefix(deployed.Image, "my.registry.domain:"), "the image pushed to the BYO registry: %q", deployed.Image) @@ -612,7 +530,7 @@ func (s *Suite) TestAirflowFailureForNoBYORegistryDomain() { }, }, nil).Once() - _, err := Airflow(s.houstonMock, "./testfiles/", "test-deployment-id", "test-workspace-id", false, false, description, false, "", Options{}) + _, err := Airflow(s.houstonMock, "test-deployment-id", "test-workspace-id", false, false, "img:1", Options{}) s.ErrorIs(err, ErrBYORegistryDomainNotSet) } @@ -621,7 +539,8 @@ func (s *Suite) TestAirflowSuccessForImageOnly() { config.InitConfig(s.fsForLocalConfig) imageHandlerInit = func(image string) airflow.ImageHandler { - s.mockImageHandler.On("Build", mock.Anything, mock.Anything, mock.Anything).Return(nil) + s.mockImageHandler.On("TagLocalImage", "img:1").Return(nil) + s.mockImageHandler.On("GetLabel", "", airflow.RuntimeImageLabel).Return("4.2.5", nil) s.mockImageHandler.On("Push", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return("", nil).Once() return s.mockImageHandler } @@ -657,7 +576,7 @@ func (s *Suite) TestAirflowSuccessForImageOnly() { vars["clusterId"] = "test-cluster-id" s.houstonMock.On("GetRuntimeReleases", vars).Return(mockRuntimeReleases, nil) - _, err := Airflow(s.houstonMock, "./testfiles/", "test-deployment-id", "test-workspace-id", false, false, description, true, "", Options{}) + _, err := Airflow(s.houstonMock, "test-deployment-id", "test-workspace-id", false, true, "img:1", Options{}) s.NoError(err) } @@ -702,7 +621,7 @@ func (s *Suite) TestAirflowSuccessForImageName() { vars["clusterId"] = "test-cluster-id" s.houstonMock.On("GetRuntimeReleases", vars).Return(mockRuntimeReleases, nil) - _, err := Airflow(s.houstonMock, "./testfiles/", "test-deployment-id", "test-workspace-id", false, false, description, true, customImageName, Options{}) + _, err := Airflow(s.houstonMock, "test-deployment-id", "test-workspace-id", false, true, customImageName, Options{}) s.NoError(err) } @@ -728,14 +647,15 @@ func (s *Suite) TestAirflowFailForImageNameWhenImageHasNoRuntimeLabel() { s.houstonMock.On("GetDeployment", "test-deployment-id").Return(deployment, nil).Once() s.houstonMock.On("GetAppConfig", mock.Anything).Return(&houston.AppConfig{}, nil).Once() - _, err := Airflow(s.houstonMock, "./testfiles/", "test-deployment-id", "test-workspace-id", false, false, description, true, customImageName, Options{}) + _, err := Airflow(s.houstonMock, "test-deployment-id", "test-workspace-id", false, true, customImageName, Options{}) s.Error(err, ErrNoRuntimeLabelOnCustomImage) } func (s *Suite) TestAirflowFailureForImageOnly() { config.InitConfig(s.fsForLocalConfig) imageHandlerInit = func(image string) airflow.ImageHandler { - s.mockImageHandler.On("Build", mock.Anything, mock.Anything, mock.Anything).Return(nil) + s.mockImageHandler.On("TagLocalImage", "img:1").Return(nil) + s.mockImageHandler.On("GetLabel", "", airflow.RuntimeImageLabel).Return("4.2.5", nil) s.mockImageHandler.On("Push", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return("", nil) return s.mockImageHandler } @@ -753,7 +673,7 @@ func (s *Suite) TestAirflowFailureForImageOnly() { s.houstonMock.On("GetDeployment", "test-deployment-id").Return(deployment, nil).Once() s.houstonMock.On("GetAppConfig", mock.Anything).Return(&houston.AppConfig{}, nil).Once() - _, err := Airflow(s.houstonMock, "./testfiles/", "test-deployment-id", "test-workspace-id", false, false, description, true, "", Options{}) + _, err := Airflow(s.houstonMock, "test-deployment-id", "test-workspace-id", false, true, "img:1", Options{}) s.Error(err, ErrDeploymentTypeIncorrectForImageOnly) } @@ -1608,7 +1528,7 @@ func (s *Suite) TestAirflowImageOnlyRefusesImageDeployments() { if !tc.refused { s.mockImageHandler.On("TagLocalImage", "img:1").Return(stop).Once() } - _, err := Airflow(s.houstonMock, "./testfiles/", "test-deployment-id", "test-workspace-id", false, false, description, true, "img:1", Options{Progress: io.Discard}) + _, err := Airflow(s.houstonMock, "test-deployment-id", "test-workspace-id", false, true, "img:1", Options{Progress: io.Discard}) if tc.refused { s.ErrorIs(err, ErrDeploymentTypeIncorrectForImageOnly) return diff --git a/internal/platform/apc/deploy/testfiles/Dockerfile b/internal/platform/apc/deploy/testfiles/Dockerfile deleted file mode 100644 index 65be93b27..000000000 --- a/internal/platform/apc/deploy/testfiles/Dockerfile +++ /dev/null @@ -1 +0,0 @@ -FROM quay.io/astronomer/ap-airflow:1.10.12-onbuild diff --git a/internal/platform/apc/deploy/testfiles/Dockerfile.invalid b/internal/platform/apc/deploy/testfiles/Dockerfile.invalid deleted file mode 100644 index 24899cb5c..000000000 --- a/internal/platform/apc/deploy/testfiles/Dockerfile.invalid +++ /dev/null @@ -1,2 +0,0 @@ -FROM quay.io/astronomer/astro-runtime:3.0.2 -CMD ["echo", 1] diff --git a/internal/platform/apc/deploy/testfiles/Dockerfile.privateImageRepo b/internal/platform/apc/deploy/testfiles/Dockerfile.privateImageRepo deleted file mode 100644 index 657fe64aa..000000000 --- a/internal/platform/apc/deploy/testfiles/Dockerfile.privateImageRepo +++ /dev/null @@ -1 +0,0 @@ -FROM quay.io/test-org/airflow:2.1.0-onbuild diff --git a/internal/platform/apc/deploy/testfiles/Dockerfile.warning b/internal/platform/apc/deploy/testfiles/Dockerfile.warning deleted file mode 100644 index a0f793fa0..000000000 --- a/internal/platform/apc/deploy/testfiles/Dockerfile.warning +++ /dev/null @@ -1 +0,0 @@ -FROM quay.io/astronomer/ap-airflow:4.5.6 diff --git a/internal/platform/astro/deploy/bundle.go b/internal/platform/astro/deploy/bundle.go index a73d67bce..fa79ea9d3 100644 --- a/internal/platform/astro/deploy/bundle.go +++ b/internal/platform/astro/deploy/bundle.go @@ -30,6 +30,9 @@ type DeployBundleInput struct { BundleType string Description string AstroV1Client astrov1.APIClient + // Login is the Astro login the deploy runs under, whose host the + // Deployment lives on; nil is the current context. + Login *config.Context } // BundleGit is the commit a bundle deploy recorded, when the bundle sits in a @@ -58,7 +61,7 @@ type BundleDeploy struct { // finalizes the deploy. It does not report the result: its caller renders the // BundleDeploy it returns. func DeployBundle(input *DeployBundleInput) (BundleDeploy, error) { - c, err := config.GetCurrentContext() + c, err := loginOrCurrent(input.Login) if err != nil { return BundleDeploy{}, err } @@ -151,6 +154,12 @@ func WaitForBundle(progress io.Writer, deploymentID string, waitTime time.Durati return deployment.HealthPoll(progress, deploymentID, dagOnlyDeploySleepTime, tickNum, int(waitTime.Seconds()), astroV1Client) } +// WaitForBundleIn is WaitForBundle under login, for a Deployment on a host +// other than the current context's. +func WaitForBundleIn(progress io.Writer, login *config.Context, deploymentID string, waitTime time.Duration, astroV1Client astrov1.APIClient) error { + return deployment.HealthPollIn(progress, login.Organization, login.Token, deploymentID, dagOnlyDeploySleepTime, tickNum, int(waitTime.Seconds()), astroV1Client) +} + type DeleteBundleInput struct { MountPath string DeploymentID string diff --git a/internal/platform/astro/deploy/cosmosboost_hook_test.go b/internal/platform/astro/deploy/cosmosboost_hook_test.go index df8c1209f..26c2bcea8 100644 --- a/internal/platform/astro/deploy/cosmosboost_hook_test.go +++ b/internal/platform/astro/deploy/cosmosboost_hook_test.go @@ -8,11 +8,8 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" - "github.com/astronomer/astro-cli/airflow" - "github.com/astronomer/astro-cli/airflow/mocks" "github.com/astronomer/astro-cli/config" astrov1_mocks "github.com/astronomer/astro-cli/internal/platform/astro/clients/astrov1/mocks" testUtil "github.com/astronomer/astro-cli/pkg/testing" @@ -149,30 +146,6 @@ func TestDeployBundleDbtPathRunsPreDeployWhenEnabled(t *testing.T) { "astro dbt deploy must still run the pre-deploy step after the hook moved into UploadBundle") } -func TestBuildImageRunsPreDeployOnBuildContextWhenEnabled(t *testing.T) { - setupCosmosBoostEnv(t) - require.NoError(t, config.CFG.CosmosBoostPreDeploy.SetHomeString("true")) - - projectDir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(projectDir, "dbt_project.yml"), []byte("name: shop\n"), 0o644)) - - // Fail the build immediately: the assertion below then proves the step ran - // against the build context BEFORE docker build. - mockImageHandler := new(mocks.ImageHandler) - airflowImageHandler = func(image string) airflow.ImageHandler { - mockImageHandler.On("Build", mock.Anything, mock.Anything, mock.Anything).Return(errMock).Once() - return mockImageHandler - } - mockV1Client := new(astrov1_mocks.ClientWithResponsesInterface) - - _, err := buildImage(projectDir, "4.2.5", "", "", "", nil, false, false, mockV1Client) - assert.ErrorIs(t, err, errMock) - - assert.FileExists(t, filepath.Join(projectDir, cosmosBoostArtifact), - "the pre-deploy step must run against the build context before docker build") - mockImageHandler.AssertExpectations(t) -} - // TestUploadBundleFailsWhenStaleArtifactUnremovable pins the safety property // of ENABLED deploys: a deploy that cannot guarantee the payload is free of // stale artifacts must fail rather than ship one, because consumers cannot @@ -224,38 +197,6 @@ func TestUploadBundleIsANoOpWhenDisabled(t *testing.T) { "a disabled deploy must not touch the tree; leftovers are cleaned by astro dbt cleanup") } -// TestBuildImageFailsWhenStaleArtifactUnremovable mirrors the bundle-path -// safety test for the image path: an enabled build that cannot guarantee the -// context is free of stale artifacts must fail before docker build runs. -func TestBuildImageFailsWhenStaleArtifactUnremovable(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("directory write-permission semantics differ on windows") - } - setupCosmosBoostEnv(t) - require.NoError(t, config.CFG.CosmosBoostPreDeploy.SetHomeString("true")) - - projectDir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(projectDir, "dbt_project.yml"), []byte("name: shop\n"), 0o644)) - stale := filepath.Join(projectDir, cosmosBoostArtifact) - require.NoError(t, os.MkdirAll(filepath.Dir(stale), 0o755)) - require.NoError(t, os.WriteFile(stale, []byte(`{"generated_by": {"application": "astro"}}`), 0o644)) - require.NoError(t, os.Chmod(filepath.Dir(stale), 0o555)) - t.Cleanup(func() { _ = os.Chmod(filepath.Dir(stale), 0o755) }) - - // No Build expectation: the failure must happen before docker build. - mockImageHandler := new(mocks.ImageHandler) - airflowImageHandler = func(image string) airflow.ImageHandler { - return mockImageHandler - } - mockV1Client := new(astrov1_mocks.ClientWithResponsesInterface) - - _, err := buildImage(projectDir, "4.2.5", "", "", "", nil, false, false, mockV1Client) - - require.Error(t, err, "the build must not proceed with a stale artifact it could not remove") - assert.ErrorContains(t, err, "Cosmos Boost") - mockImageHandler.AssertExpectations(t) -} - // TestUploadBundleFailsOnForeignSidecar: an enabled deploy must not ship a // sidecar from an unrecognized producer - cleanup will not delete it, the // plugin would consume it, so the deploy stops and asks for manual removal. diff --git a/internal/platform/astro/deploy/deploy.go b/internal/platform/astro/deploy/deploy.go index 121bf4cfa..5d50e2803 100644 --- a/internal/platform/astro/deploy/deploy.go +++ b/internal/platform/astro/deploy/deploy.go @@ -1,11 +1,8 @@ package deploy import ( - "bufio" - "bytes" httpContext "context" "fmt" - "io" "os" "path/filepath" "strings" @@ -21,64 +18,27 @@ import ( "github.com/astronomer/astro-cli/config" "github.com/astronomer/astro-cli/docker" "github.com/astronomer/astro-cli/internal/platform/astro/clients/astrov1" - astrov1alpha1 "github.com/astronomer/astro-cli/internal/platform/astro/clients/astrov1alpha1" "github.com/astronomer/astro-cli/internal/platform/astro/deployment" - "github.com/astronomer/astro-cli/internal/platform/astro/organization" - "github.com/astronomer/astro-cli/pkg/ansi" "github.com/astronomer/astro-cli/pkg/azure" - "github.com/astronomer/astro-cli/pkg/cosmosboost" "github.com/astronomer/astro-cli/pkg/fileutil" - "github.com/astronomer/astro-cli/pkg/httputil" - "github.com/astronomer/astro-cli/pkg/input" - "github.com/astronomer/astro-cli/pkg/logger" "github.com/astronomer/astro-cli/pkg/util" ) -// noDagsPrompt is asked before a deploy with no DAGs replaces the existing ones. -const noDagsPrompt = "Warning: No Dags found. This will delete any existing Dags. Are you sure you want to deploy?" - const ( - parse = "parse" - astroDomain = "astronomer.io" - registryUsername = "cli" - runtimeImageLabel = airflow.RuntimeImageLabel - dagParseAllowedVersion = "4.1.0" - - composeImageBuildingPromptMsg = "Building image..." - composeSkipImageBuildingPromptMsg = "Skipping building image..." - deploymentHeaderMsg = "Authenticated to %s \n\n" - - warningInvalidImageNameMsg = "WARNING! The image in your Dockerfile '%s' is not based on Astro Runtime and is not supported. Change your Dockerfile with an image that pulls from 'quay.io/astronomer/astro-runtime' to proceed.\n" - warningInvalidPrebuiltImageNameMsg = "WARNING! The image '%s' does not appear to be based on Astro Runtime (the '%s' label is missing). Ensure your image is built FROM quay.io/astronomer/astro-runtime to proceed.\n" - - allTests = "all-tests" - parseAndPytest = "parse-and-all-tests" + registryUsername = "cli" + runtimeImageLabel = airflow.RuntimeImageLabel enableDagDeployMsg = "Dag-only deploys are not enabled for this Deployment. Run 'astro deployment update %s --dag-deploy enable' to enable Dag-only deploys" dagDeployDisabled = "dag deploy is not enabled for deployment" - workspaceFlagMismatchMsg = "deployment %s is in workspace %s, not workspace %s given by --workspace. Pass --workspace %s, or leave the flag out" - otherWorkspaceMsg = "Deployment %s is in workspace %s, not the current workspace %s. Deploying to it there.\n" errCiCdEnforcementUpdate = "cannot deploy since ci/cd enforcement is enabled for the deployment %s. Please use API Tokens instead" ) var ( - pytestFile string - dockerfile = "Dockerfile" - deployImagePlatformSupport = []string{"linux/amd64"} // Monkey patched to write unit tests airflowImageHandler = airflow.ImageHandlerInit - dagCheckInit = airflow.DAGCheckInit azureUploader = azure.Upload canCiCdDeploy = deployment.CanCiCdDeploy - dagTarballVersion = "" - dagsUploadURL = "" - nextTag = "" -) - -var ( - errDagsParseFailed = errors.New("your local Dags did not parse. Fix the listed errors or use `astro deploy [deployment-id] -f` to force deploy") //nolint:revive // intentional in this shell code - envFileMissing = errors.New("Env file path is incorrect: ") //nolint:revive // intentional in this shell code ) var ( @@ -88,45 +48,18 @@ var ( ) type deploymentInfo struct { - deploymentID string namespace string deployImage string currentVersion string organizationID string workspaceID string webserverURL string - deploymentType string desiredDagTarballVersion string dagDeployEnabled bool cicdEnforcement bool - name string isRemoteExecutionEnabled bool } -type InputDeploy struct { - Path string - RuntimeID string - WsID string - WsIDFromFlag bool - Pytest string - EnvFile string - ImageName string - DeploymentName string - Prompt bool - Dags bool - NoDagsBaseDir bool - Image bool - WaitForStatus bool - WaitTime time.Duration - // Progress takes the --wait progress; nil is stderr. - Progress io.Writer - DagsPath string - Description string - BuildSecrets []string - Force bool - DagBundleName string -} - // InputClientDeploy contains inputs for client image deployments type InputClientDeploy struct { Path string @@ -136,60 +69,12 @@ type InputClientDeploy struct { DeploymentID string } -const accessYourDeploymentFmt = ` - - Access your Deployment: - - Deployment View: %s - Airflow UI: %s -` - -func removeDagsFromDockerIgnore(fullpath string) error { - original, err := os.ReadFile(fullpath) - if err != nil { - return err - } - - hadTrailingNewline := len(original) > 0 && original[len(original)-1] == '\n' - - var buf bytes.Buffer - scanner := bufio.NewScanner(bytes.NewReader(original)) - for scanner.Scan() { - text := scanner.Text() - if text != "dags/" { - _, err = buf.WriteString(text + "\n") - if err != nil { - return err - } - } - } - - if err := scanner.Err(); err != nil { - return err - } - - result := bytes.TrimRight(buf.Bytes(), "\n") - if hadTrailingNewline && len(result) > 0 { - result = append(result, '\n') - } - - return os.WriteFile(fullpath, result, 0o666) //nolint:gosec,mnd // rewrites the user's own .dockerignore in place -} - -func shouldIncludeMonitoringDag(deploymentType astrov1.DeploymentType) bool { - return includeMonitoringDag(organization.IsOrgHosted(), deploymentType) -} - -// includeMonitoringDag is shouldIncludeMonitoringDag for an org whose product -// is already known, which need not be the current context's. +// includeMonitoringDag reports whether a DAG upload carries the monitoring DAG, +// for an org whose product is known, which need not be the current context's. func includeMonitoringDag(orgHosted bool, deploymentType astrov1.DeploymentType) bool { return !orgHosted && !deployment.IsDeploymentDedicated(deploymentType) && !deployment.IsDeploymentStandard(deploymentType) } -func deployDags(path, dagsPath, dagsUploadURL, currentRuntimeVersion string, deploymentType astrov1.DeploymentType, noDagsBaseDir bool) (string, error) { - return uploadDags(path, dagsPath, dagsUploadURL, currentRuntimeVersion, shouldIncludeMonitoringDag(deploymentType), noDagsBaseDir) -} - func uploadDags(path, dagsPath, dagsUploadURL, currentRuntimeVersion string, monitoringDag, noDagsBaseDir bool) (string, error) { if monitoringDag { monitoringDagPath := filepath.Join(dagsPath, "astronomer_monitoring_dag.py") @@ -225,421 +110,6 @@ func uploadDags(path, dagsPath, dagsUploadURL, currentRuntimeVersion string, mon return versionID, nil } -// Deploy pushes a new docker image -func Deploy(deployInput InputDeploy, astroV1Client astrov1.APIClient, astroV1Alpha1Client astrov1alpha1.APIClient) error { //nolint:gocognit,gocyclo,gocritic // v1 complexity, refactor tracked separately - c, err := config.GetCurrentContext() - if err != nil { - return err - } - - if c.Domain == astroDomain { - fmt.Printf(deploymentHeaderMsg, "Astro") - } else { - fmt.Printf(deploymentHeaderMsg, c.Domain) - } - - deployInfo, err := getDeploymentInfo(deployInput.RuntimeID, deployInput.WsID, deployInput.DeploymentName, deployInput.Prompt, astroV1Client) - if err != nil { - return err - } - - var dagsPath string - if deployInput.DagsPath != "" { - dagsPath = deployInput.DagsPath - } else { - dagsPath = filepath.Join(deployInput.Path, "dags") - } - - var dagFiles []string - if !deployInfo.isRemoteExecutionEnabled { - dagFiles = fileutil.GetFilesWithSpecificExtension(dagsPath, ".py") - } - - if deployInfo.cicdEnforcement { - if !canCiCdDeploy(c.Token) { - return fmt.Errorf(errCiCdEnforcementUpdate, deployInfo.name) - } - } - - if deployInput.WsID != deployInfo.workspaceID { - if deployInput.WsIDFromFlag { - return fmt.Errorf(workspaceFlagMismatchMsg, deployInfo.deploymentID, deployInfo.workspaceID, deployInput.WsID, deployInfo.workspaceID) - } - fmt.Printf(otherWorkspaceMsg, deployInfo.deploymentID, deployInfo.workspaceID, deployInput.WsID) - } - - if deployInput.Image && !deployInfo.isRemoteExecutionEnabled { - if !deployInfo.dagDeployEnabled { - return fmt.Errorf(enableDagDeployMsg, deployInfo.deploymentID) - } - } - - deploymentURL, err := deployment.GetDeploymentURL(deployInfo.deploymentID, deployInfo.workspaceID) - if err != nil { - return err - } - - // Check if git metadata is enabled (default: true). - // Skip when --image-name is provided: the local working directory does not necessarily - // reflect the contents of a prebuilt image, so attaching its git metadata would be misleading. - var deployGit *astrov1.CreateDeployGitRequest - var commitMessage string - if config.CFG.DeployGitMetadata.GetBool() && deployInput.ImageName == "" { - deployGit, commitMessage = retrieveLocalGitMetadata(deployInput.Path) - } - - // Use commit message as description fallback - description := deployInput.Description - if description == "" { - description = commitMessage - } - - // Build the deploy request with git metadata - createDeployRequest := astrov1.CreateDeployRequest{ - Description: &description, - } - - // Set deploy type - switch { - case deployInput.Dags: - createDeployRequest.Type = astrov1.CreateDeployRequestTypeDAGONLY - case deployInput.Image: - createDeployRequest.Type = astrov1.CreateDeployRequestTypeIMAGEONLY - default: - createDeployRequest.Type = astrov1.CreateDeployRequestTypeIMAGEANDDAG - } - - createDeployRequest.Git = deployGit - - var deployID, imageRepository string - if deployInput.DagBundleName != "" { - // dagBundleName exists only on the v1alpha1 deploy API, so route just the - // create call there; the upload and finalize steps below stay on v1 and - // operate on the same underlying deploy. This cross-API bridge is - // intentional and temporary — collapse back into the v1 create once - // dagBundleName reaches the v1 deploy API. - deploy, err := createDeployWithDagBundle(deployInfo.organizationID, deployInfo.deploymentID, description, deployInput.DagBundleName, deployInput.Dags, deployGit, astroV1Alpha1Client) - if err != nil { - return err - } - deployID = deploy.Id - imageRepository = deploy.ImageRepository - if deploy.DagsUploadUrl != nil { - dagsUploadURL = *deploy.DagsUploadUrl - } else { - dagsUploadURL = "" - } - if deploy.ImageTag != "" { - nextTag = deploy.ImageTag - } else { - nextTag = "" - } - } else { - deploy, err := createDeploy(deployInfo.organizationID, deployInfo.deploymentID, createDeployRequest, astroV1Client) - if err != nil { - return err - } - deployID = deploy.Id - imageRepository = deploy.ImageRepository - if deploy.DagsUploadUrl != nil { - dagsUploadURL = *deploy.DagsUploadUrl - } else { - dagsUploadURL = "" - } - if deploy.ImageTag != "" { - nextTag = deploy.ImageTag - } else { - nextTag = "" - } - } - - if deployInput.Dags { - if len(dagFiles) == 0 && config.CFG.ShowWarnings.GetBool() && !deployInput.Force { - i, err := input.Confirm(noDagsPrompt, input.AnsweredBy("--force")) - if err != nil { - return err - } - - if !i { - fmt.Println("Canceling deploy...") - return nil - } - } - if deployInput.Pytest != "" { - runtimeVersion, err := buildImage(deployInput.Path, deployInfo.currentVersion, deployInfo.deployImage, deployInput.ImageName, deployInfo.organizationID, deployInput.BuildSecrets, deployInfo.dagDeployEnabled, deployInfo.isRemoteExecutionEnabled, astroV1Client) - if err != nil { - return err - } - - err = parseOrPytestDAG(deployInput.Pytest, runtimeVersion, deployInput.EnvFile, deployInfo.deployImage, deployInfo.namespace, deployInput.BuildSecrets) - if err != nil { - return err - } - } - - if !deployInfo.dagDeployEnabled { - return fmt.Errorf(enableDagDeployMsg, deployInfo.deploymentID) - } - - fmt.Println("Initiating Dag deploy for: " + deployInfo.deploymentID) - dagTarballVersion, err = deployDags(deployInput.Path, dagsPath, dagsUploadURL, deployInfo.currentVersion, astrov1.DeploymentType(deployInfo.deploymentType), deployInput.NoDagsBaseDir) - if err != nil { - if strings.Contains(err.Error(), dagDeployDisabled) { - return fmt.Errorf(enableDagDeployMsg, deployInfo.deploymentID) - } - - return err - } - - // finish deploy - err = finalizeDeploy(deployID, deployInfo.deploymentID, deployInfo.organizationID, dagTarballVersion, deployInfo.dagDeployEnabled, astroV1Client) - if err != nil { - return err - } - - if deployInput.WaitForStatus { - // Keeping wait timeout low since dag only deploy is faster - err = deployment.HealthPoll(deployInput.Progress, deployInfo.deploymentID, dagOnlyDeploySleepTime, tickNum, int(deployInput.WaitTime.Seconds()), astroV1Client) - if err != nil { - return err - } - - fmt.Println( - "\nSuccessfully uploaded Dags with version " + ansi.Bold(dagTarballVersion) + " to Astro. Navigate to the Airflow UI to confirm that your deploy was successful." + - fmt.Sprintf(accessYourDeploymentFmt, ansi.Bold(deploymentURL), ansi.Bold(deployInfo.webserverURL)), - ) - - return nil - } - - fmt.Println( - "\nSuccessfully uploaded Dags with version " + ansi.Bold( - dagTarballVersion, - ) + " to Astro. Navigate to the Airflow UI to confirm that your deploy was successful. The Airflow UI takes about 1 minute to update." + - fmt.Sprintf( - accessYourDeploymentFmt, - ansi.Bold(deploymentURL), - ansi.Bold(deployInfo.webserverURL), - ), - ) - } else { - fullpath := filepath.Join(deployInput.Path, ".dockerignore") - fileExist, _ := fileutil.Exists(fullpath, nil) //nolint:errcheck // treated as absent on error - if fileExist { - err := removeDagsFromDockerIgnore(fullpath) - if err != nil { - return errors.Wrap(err, "Found dags entry in .dockerignore file. Remove this entry and try again") - } - } - envFileExists, _ := fileutil.Exists(deployInput.EnvFile, nil) //nolint:errcheck // treated as absent on error - if !envFileExists && deployInput.EnvFile != ".env" { - return fmt.Errorf("%w %s", envFileMissing, deployInput.EnvFile) - } - - if deployInfo.dagDeployEnabled && len(dagFiles) == 0 && config.CFG.ShowWarnings.GetBool() && !deployInput.Image && !deployInput.Force { - i, err := input.Confirm(noDagsPrompt, input.AnsweredBy("--force")) - if err != nil { - return err - } - - if !i { - fmt.Println("Canceling deploy...") - return nil - } - } - - // Build our image - runtimeVersion, err := buildImage(deployInput.Path, deployInfo.currentVersion, deployInfo.deployImage, deployInput.ImageName, deployInfo.organizationID, deployInput.BuildSecrets, deployInfo.dagDeployEnabled, deployInfo.isRemoteExecutionEnabled, astroV1Client) - if err != nil { - return err - } - - if len(dagFiles) > 0 { - err = parseOrPytestDAG(deployInput.Pytest, runtimeVersion, deployInput.EnvFile, deployInfo.deployImage, deployInfo.namespace, deployInput.BuildSecrets) - if err != nil { - return err - } - } else { - fmt.Println("No Dags found. Skipping testing...") - } - - repository := imageRepository - // TODO: Resolve the edge case where two people push the same nextTag at the same time - remoteImage := fmt.Sprintf("%s:%s", repository, nextTag) - - imageHandler := airflowImageHandler(deployInfo.deployImage) - fmt.Println("Pushing image to Astronomer registry") - _, err = imageHandler.Push(remoteImage, registryUsername, c.Token, false) - if err != nil { - return err - } - - if deployInfo.dagDeployEnabled && len(dagFiles) > 0 { - if !deployInput.Image { - dagTarballVersion, err = deployDags(deployInput.Path, dagsPath, dagsUploadURL, deployInfo.currentVersion, astrov1.DeploymentType(deployInfo.deploymentType), deployInput.NoDagsBaseDir) - if err != nil { - return err - } - } else { - fmt.Println("Image Deploy only. Skipping deploying Dag...") - } - } - // finish deploy - err = finalizeDeploy(deployID, deployInfo.deploymentID, deployInfo.organizationID, dagTarballVersion, deployInfo.dagDeployEnabled, astroV1Client) - if err != nil { - return err - } - - if deployInput.WaitForStatus { - err = deployment.HealthPoll(deployInput.Progress, deployInfo.deploymentID, sleepTime, tickNum, int(deployInput.WaitTime.Seconds()), astroV1Client) - if err != nil { - return err - } - } - - fmt.Println("Successfully pushed image to Astronomer registry. Navigate to the Astronomer UI for confirmation that your deploy was successful. To deploy dags only run astro deploy --dags." + - fmt.Sprintf(accessYourDeploymentFmt, ansi.Bold("https://"+deploymentURL), ansi.Bold("https://"+deployInfo.webserverURL))) - } - - return nil -} - -func getDeploymentInfo( - deploymentID, wsID, deploymentName string, - prompt bool, - astroV1Client astrov1.APIClient, -) (deploymentInfo, error) { - // Use config deployment if provided - if deploymentID == "" { - deploymentID = config.CFG.ProjectDeployment.GetProjectString() - if deploymentID != "" { - fmt.Printf("Deployment ID found in the config file. This Deployment ID will be used for the deploy\n") - } - } - - if deploymentID != "" && deploymentName != "" { - fmt.Printf("Both a Deployment ID and Deployment name have been supplied. The Deployment ID %s will be used for the Deploy\n", deploymentID) - } - - // check if deploymentID or if force prompt was requested was given by user - if deploymentID == "" || prompt { - currentDeployment, err := deployment.GetDeployment(wsID, deploymentID, deploymentName, false, nil, astroV1Client) - if err != nil { - return deploymentInfo{}, err - } - deploymentByID, err := deployment.GetDeploymentByID(currentDeployment.OrganizationId, currentDeployment.Id, astroV1Client) - if err != nil { - return deploymentInfo{}, err - } - var desiredDagTarballVersion string - if deploymentByID.DesiredDagTarballVersion != nil { - desiredDagTarballVersion = *deploymentByID.DesiredDagTarballVersion - } else { - desiredDagTarballVersion = "" - } - - return deploymentInfo{ - currentDeployment.Id, - currentDeployment.Namespace, - airflow.ImageName(currentDeployment.Namespace, "latest"), - currentDeployment.AstroRuntimeVersion, - currentDeployment.OrganizationId, - currentDeployment.WorkspaceId, - currentDeployment.WebServerUrl, - string(*currentDeployment.Type), - desiredDagTarballVersion, - currentDeployment.IsDagDeployEnabled, - currentDeployment.IsCicdEnforced, - currentDeployment.Name, - deployment.IsRemoteExecutionEnabled(¤tDeployment), - }, nil - } - c, err := config.GetCurrentContext() - if err != nil { - return deploymentInfo{}, err - } - deployInfo, err := fetchDeploymentDetails(deploymentID, c.Organization, astroV1Client) - if err != nil { - return deploymentInfo{}, err - } - deployInfo.deploymentID = deploymentID - return deployInfo, nil -} - -func parseOrPytestDAG(pytest, runtimeVersion, envFile, deployImage, namespace string, buildSecrets []string) error { - validDAGParseVersion := airflowversions.CompareRuntimeVersions(runtimeVersion, dagParseAllowedVersion) >= 0 - if !validDAGParseVersion { - fmt.Println("\nruntime image is earlier than 4.1.0, this deploy will skip Dag parse...") - } - - dagCheck, err := dagCheckInit(config.WorkingPath, envFile, "Dockerfile", namespace) - if err != nil { - return err - } - - switch { - case pytest == parse && validDAGParseVersion: - // parse dags - fmt.Println("Testing image...") - err := parseDAGs(deployImage, buildSecrets, dagCheck) - if err != nil { - return err - } - case pytest != "" && pytest != parse && pytest != parseAndPytest: - // check pytests - fmt.Println("Testing image...") - err := checkPytest(pytest, deployImage, buildSecrets, dagCheck) - if err != nil { - return err - } - case pytest == parseAndPytest: - // parse dags and check pytests - fmt.Println("Testing image...") - err := parseDAGs(deployImage, buildSecrets, dagCheck) - if err != nil { - return err - } - - err = checkPytest(pytest, deployImage, buildSecrets, dagCheck) - if err != nil { - return err - } - } - return nil -} - -func parseDAGs(deployImage string, buildSecrets []string, dagCheck airflow.DAGCheck) error { - if !config.CFG.SkipParse.GetBool() && !util.CheckEnvBool(os.Getenv("ASTRONOMER_SKIP_PARSE")) { - err := dagCheck.Parse("", deployImage, buildSecrets) - if err != nil { - fmt.Println(err) - return errDagsParseFailed - } - } else { - fmt.Println("Skipping parsing dags due to skip parse being set to true in either the config.yaml or local environment variables") - } - - return nil -} - -// Validate code with pytest -func checkPytest(pytest, deployImage string, buildSecrets []string, dagCheck airflow.DAGCheck) error { - if pytest != allTests && pytest != parseAndPytest { - pytestFile = pytest - } - - exitCode, err := dagCheck.Pytest(pytestFile, "", deployImage, "", buildSecrets) - if err != nil { - if strings.Contains(exitCode, "1") { // exit code is 1 meaning tests failed - return errors.New("at least 1 pytest in your tests directory failed. Fix the issues listed or rerun the command without the '--pytest' flag to deploy") - } - return errors.Wrap(err, "Something went wrong while Pytesting your Dags,\nif the issue persists rerun the command without the '--pytest' flag to deploy") - } - - fmt.Print("\nAll Pytests passed!\n") - return err -} - func fetchDeploymentDetails(deploymentID, organizationID string, astroV1Client astrov1.APIClient) (deploymentInfo, error) { resp, err := astroV1Client.GetDeploymentWithResponse(httpContext.Background(), organizationID, deploymentID) if err != nil { @@ -682,172 +152,6 @@ func fetchDeploymentDetails(deploymentID, organizationID string, astroV1Client a }, nil } -func buildImageWithoutDags(path string, buildSecrets []string, imageHandler airflow.ImageHandler) (err error) { - fullpath := filepath.Join(path, ".dockerignore") - - // Snapshot the original bytes so we can restore byte-for-byte after the build - // (preserves CRLF, trailing whitespace, etc.). - originalBytes, err := os.ReadFile(fullpath) - originalExisted := err == nil - if err != nil && !os.IsNotExist(err) { - return err - } - - defer func() { - var restoreErr error - if originalExisted { - restoreErr = os.WriteFile(fullpath, originalBytes, 0o644) //nolint:gosec,mnd // restore the user's own .dockerignore with its standard perms - } else { - restoreErr = os.Remove(fullpath) - } - // A failed restore leaves the user's .dockerignore modified, so report - // it. Don't mask an earlier build error. - if restoreErr != nil && err == nil { - err = fmt.Errorf("restoring %s after build: %w", fullpath, restoreErr) - } - }() - - switch { - case !originalExisted: - if err := os.WriteFile(fullpath, []byte("dags/\n"), 0o644); err != nil { //nolint:gosec,mnd // temporary .dockerignore for the dagless build, standard perms - return err - } - case !dockerignoreContainsDags(originalBytes): - modified := append([]byte{}, originalBytes...) - if len(modified) > 0 && modified[len(modified)-1] != '\n' { - modified = append(modified, '\n') - } - modified = append(modified, []byte("dags/\n")...) - if err := os.WriteFile(fullpath, modified, 0o644); err != nil { //nolint:gosec,mnd // rewrite the user's .dockerignore with its standard perms - return err - } - } - - return imageHandler.Build("", buildSecrets, types.ImageBuildConfig{Path: path, TargetPlatforms: deployImagePlatformSupport}) -} - -// dockerignoreContainsDags reports whether content has a line equal to "dags/". -// Uses bufio.Scanner so CRLF line endings are handled identically to LF. -func dockerignoreContainsDags(content []byte) bool { - scanner := bufio.NewScanner(bytes.NewReader(content)) - for scanner.Scan() { - if scanner.Text() == "dags/" { - return true - } - } - return false -} - -func buildImage(path, currentVersion, deployImage, imageName, organizationID string, buildSecrets []string, dagDeployEnabled, isRemoteExecutionEnabled bool, astroV1Client astrov1.APIClient) (version string, err error) { - imageHandler := airflowImageHandler(deployImage) - - if imageName == "" { - // Build our image - fmt.Println(composeImageBuildingPromptMsg) - - // Cosmos Boost pre-deploy step, opt-in via cosmos_boost.pre_deploy; - // with the setting off the build does not touch the tree at all. - // Cleanup runs first and is fatal on failure (a stale artifact must - // not ship inside the image), while stamping is best-effort (a - // missing artifact is safe). Artifacts left by earlier enabled - // deploys are removed with `astro dbt cleanup`. - if config.CFG.CosmosBoostPreDeploy.GetBool() { - if err := cosmosboost.EnsureClean(path); err != nil { - return "", err - } - cosmosboost.BestEffortPreDeploy(path) - } - - if dagDeployEnabled || isRemoteExecutionEnabled { - err := buildImageWithoutDags(path, buildSecrets, imageHandler) - if err != nil { - return "", err - } - } else { - err := imageHandler.Build("", buildSecrets, types.ImageBuildConfig{Path: path, TargetPlatforms: deployImagePlatformSupport}) - if err != nil { - return "", err - } - } - } else { - // skip build if an imageName is passed - fmt.Println(composeSkipImageBuildingPromptMsg) - - err := imageHandler.TagLocalImage(imageName) - if err != nil { - return "", err - } - } - - version, err = imageHandler.GetLabel("", runtimeImageLabel) - if err != nil { - fmt.Println("unable get runtime version from image") - } - - if config.CFG.ShowWarnings.GetBool() && version == "" { - if imageName != "" { - // Registry image names are arbitrary and do not convey base image - // information; reference the missing label in the warning instead. - fmt.Printf(warningInvalidPrebuiltImageNameMsg, imageName, runtimeImageLabel) - } else { - // Parse the Dockerfile to include the FROM image in the warning, - // giving the user a concrete reference for what needs to change. - cmds, err := docker.ParseFile(filepath.Join(path, dockerfile)) - if err != nil { - return "", errors.Wrapf(err, "failed to parse dockerfile: %s", filepath.Join(path, dockerfile)) - } - fmt.Printf(warningInvalidImageNameMsg, docker.GetImageFromParsedFile(cmds)) - } - fmt.Println("Canceling deploy...") - os.Exit(1) - } - - resp, err := astroV1Client.GetDeploymentOptionsWithResponse(httpContext.Background(), organizationID, &astrov1.GetDeploymentOptionsParams{}) - if err != nil { - return "", err - } - err = astrov1.NormalizeAPIError(resp.HTTPResponse, resp.Body) - if err != nil { - return "", err - } - deploymentOptionsRuntimeVersions := []string{} - for _, runtimeRelease := range resp.JSON200.RuntimeReleases { - deploymentOptionsRuntimeVersions = append(deploymentOptionsRuntimeVersions, runtimeRelease.Version) - } - - if !ValidRuntimeVersion(currentVersion, version, deploymentOptionsRuntimeVersions) { - fmt.Println("Canceling deploy...") - os.Exit(1) - } - - WarnIfNonLatestVersion(version, httputil.NewHTTPClient()) - - return version, nil -} - -// finalize deploy -func finalizeDeploy(deployID, deploymentID, organizationID, dagTarballVersion string, dagDeploy bool, astroV1Client astrov1.APIClient) error { - finalizeDeployRequest := astrov1.FinalizeDeployRequest{} - if dagDeploy { - finalizeDeployRequest.DagTarballVersion = &dagTarballVersion - } - resp, err := astroV1Client.FinalizeDeployWithResponse(httpContext.Background(), organizationID, deploymentID, deployID, finalizeDeployRequest) - if err != nil { - return err - } - err = astrov1.NormalizeAPIError(resp.HTTPResponse, resp.Body) - if err != nil { - return err - } - if resp.JSON200.DagTarballVersion != nil { - fmt.Println("Deployed Dag bundle: ", *resp.JSON200.DagTarballVersion) - } - if resp.JSON200.ImageTag != "" { - fmt.Println("Deployed Image Tag: ", resp.JSON200.ImageTag) - } - return nil -} - func createDeploy(organizationID, deploymentID string, request astrov1.CreateDeployRequest, astroV1Client astrov1.APIClient) (*astrov1.Deploy, error) { resp, err := astroV1Client.CreateDeployWithResponse(httpContext.Background(), organizationID, deploymentID, request) if err != nil { @@ -860,105 +164,6 @@ func createDeploy(organizationID, deploymentID string, request astrov1.CreateDep return resp.JSON200, err } -// createDeployWithDagBundle creates a deploy targeting a named DAG bundle via the -// v1alpha1 deploy API, the only tier that accepts dagBundleName. The v1alpha1 -// type vocabulary is simpler than v1's: the server expands IMAGE to IMAGE_AND_DAG -// (or DAG to DAG_ONLY) based on the deployment, so a plain deploy maps to IMAGE -// and a --dags deploy maps to DAG. Remove once dagBundleName reaches the v1 API. -func createDeployWithDagBundle(organizationID, deploymentID, description, dagBundleName string, dags bool, git *astrov1.CreateDeployGitRequest, client astrov1alpha1.APIClient) (*astrov1alpha1.Deploy, error) { - deployType := astrov1alpha1.CreateDeployRequestTypeIMAGE - if dags { - deployType = astrov1alpha1.CreateDeployRequestTypeDAG - } - request := astrov1alpha1.CreateDeployRequest{ - Type: deployType, - Description: &description, - DagBundleName: &dagBundleName, - Git: toV1Alpha1GitRequest(git), - } - resp, err := client.CreateDeployWithResponse(httpContext.Background(), organizationID, deploymentID, request) - if err != nil { - return nil, err - } - err = astrov1alpha1.NormalizeAPIError(resp.HTTPResponse, resp.Body) - if err != nil { - return nil, err - } - return resp.JSON200, nil -} - -func toV1Alpha1GitRequest(git *astrov1.CreateDeployGitRequest) *astrov1alpha1.CreateDeployGitRequest { - if git == nil { - return nil - } - return &astrov1alpha1.CreateDeployGitRequest{ - Account: git.Account, - AuthorName: git.AuthorName, - AuthorUrl: git.AuthorUrl, - AuthorUsername: git.AuthorUsername, - BeforeCommitSha: git.BeforeCommitSha, - Branch: git.Branch, - CommitSha: git.CommitSha, - CommitUrl: git.CommitUrl, - Path: git.Path, - Provider: astrov1alpha1.CreateDeployGitRequestProvider(git.Provider), - RemoteUrl: git.RemoteUrl, - Repo: git.Repo, - } -} - -func ValidRuntimeVersion(currentVersion, tag string, deploymentOptionsRuntimeVersions []string) bool { - // Allow old deployments which do not have runtimeVersion tag - if currentVersion == "" { - return true - } - - // Check that the tag is not a downgrade - if airflowversions.CompareRuntimeVersions(tag, currentVersion) < 0 { - fmt.Printf("Cannot deploy a downgraded Astro Runtime version. Modify your Astro Runtime version to %s or higher in your Dockerfile\n", currentVersion) - return false - } - - // Check that the tag is supported by the deployment - tagInDeploymentOptions := false - for _, runtimeVersion := range deploymentOptionsRuntimeVersions { - if airflowversions.CompareRuntimeVersions(tag, runtimeVersion) == 0 { - tagInDeploymentOptions = true - break - } - } - if !tagInDeploymentOptions { - fmt.Println("Cannot deploy an unsupported Astro Runtime version. Modify your Astro Runtime version to a supported version in your Dockerfile") - fmt.Printf("Supported versions: %s\n", strings.Join(deploymentOptionsRuntimeVersions, ", ")) - return false - } - - // If upgrading from Airflow 2 to Airflow 3, we require at least Runtime 12.0.0 (Airflow 2.10.0) - currentVersionAirflowMajorVersion := airflowversions.AirflowMajorVersionForRuntimeVersion(currentVersion) - tagAirflowMajorVersion := airflowversions.AirflowMajorVersionForRuntimeVersion(tag) - if currentVersionAirflowMajorVersion == "2" && tagAirflowMajorVersion == "3" { - if airflowversions.CompareRuntimeVersions(currentVersion, "12.0.0") < 0 { - fmt.Println("Can only upgrade deployment from Airflow 2 to Airflow 3 with deployment at Astro Runtime 12.0.0 or higher") - return false - } - } - - return true -} - -func WarnIfNonLatestVersion(version string, httpClient *httputil.HTTPClient) { - client := airflowversions.NewClient(httpClient, false, false) - latestRuntimeVersion, err := airflowversions.GetDefaultImageTag(client, "", "", false) - if err != nil { - logger.Debugf("unable to get latest runtime version: %s", err) - return - } - - if airflowversions.CompareRuntimeVersions(version, latestRuntimeVersion) < 0 { - fmt.Printf("WARNING! You are currently running Astro Runtime Version %s\nConsider upgrading to the latest version, Astro Runtime %s\n", version, latestRuntimeVersion) - } -} - // ClientBuildContext represents a prepared build context for client deployment type ClientBuildContext struct { // TempDir is the temporary directory containing the build context diff --git a/internal/platform/astro/deploy/deploy_test.go b/internal/platform/astro/deploy/deploy_test.go index 10b441ec7..666fedf9e 100644 --- a/internal/platform/astro/deploy/deploy_test.go +++ b/internal/platform/astro/deploy/deploy_test.go @@ -1,9 +1,7 @@ package deploy import ( - "bytes" "errors" - "io" "net/http" "os" "path/filepath" @@ -18,11 +16,6 @@ import ( "github.com/astronomer/astro-cli/config" "github.com/astronomer/astro-cli/internal/platform/astro/clients/astrov1" astrov1_mocks "github.com/astronomer/astro-cli/internal/platform/astro/clients/astrov1/mocks" - astrov1alpha1 "github.com/astronomer/astro-cli/internal/platform/astro/clients/astrov1alpha1" - astrov1alpha1_mocks "github.com/astronomer/astro-cli/internal/platform/astro/clients/astrov1alpha1/mocks" - "github.com/astronomer/astro-cli/internal/platform/astro/deployment" - "github.com/astronomer/astro-cli/pkg/fileutil" - "github.com/astronomer/astro-cli/pkg/httputil" testUtil "github.com/astronomer/astro-cli/pkg/testing" ) @@ -167,1287 +160,6 @@ var ( } ) -func TestDeployWithoutDagsDeploySuccess(t *testing.T) { - mockV1Client := new(astrov1_mocks.ClientWithResponsesInterface) - deployInput := InputDeploy{ - Path: "./testfiles/", - RuntimeID: "", - WsID: ws, - Pytest: "parse", - EnvFile: "./testfiles/.env", - ImageName: "", - DeploymentName: "", - Prompt: true, - WaitForStatus: false, - Dags: false, - } - testUtil.InitTestConfig(testUtil.LocalPlatform) - config.CFG.ShowWarnings.SetHomeString("false") - - mockV1Client.On("GetDeploymentWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(&deploymentResponse, nil).Times(6) - mockV1Client.On("ListDeploymentsWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(&mockListDeploymentsResponse, nil).Once() - mockV1Client.On("GetDeploymentOptionsWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(&getDeploymentOptionsResponse, nil).Times(5) - mockV1Client.On("CreateDeployWithResponse", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&createDeployResponse, nil).Times(5) - mockV1Client.On("FinalizeDeployWithResponse", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&finalizeDeployResponse, nil).Times(5) - - mockImageHandler := new(mocks.ImageHandler) - airflowImageHandler = func(image string) airflow.ImageHandler { - mockImageHandler.On("Build", mock.Anything, mock.Anything, mock.Anything).Return(nil) - mockImageHandler.On("Push", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return("", nil) - mockImageHandler.On("GetLabel", mock.Anything, runtimeImageLabel).Return("12.0.0", nil) - mockImageHandler.On("TagLocalImage", mock.Anything).Return(nil) - return mockImageHandler - } - - mockDAGCheck := new(mocks.DAGCheck) - dagCheckInit = func(airflowHome, envFile, dockerfile, imageName string) (airflow.DAGCheck, error) { - mockDAGCheck.On("Parse", mock.Anything, mock.Anything, mock.Anything).Return(nil) - mockDAGCheck.On("Pytest", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return("", nil) - return mockDAGCheck, nil - } - - ctx, err := config.GetCurrentContext() - assert.NoError(t, err) - ctx.Token = "test testing" - err = ctx.SetContext() - assert.NoError(t, err) - - // mock os.Stdin - input := []byte("1") - r, w, err := os.Pipe() - if err != nil { - t.Fatal(err) - } - _, err = w.Write(input) - if err != nil { - t.Error(err) - } - w.Close() - stdin := os.Stdin - // Restore stdin right after the test. - defer func() { os.Stdin = stdin }() - os.Stdin = r - - defer testUtil.MockUserInput(t, "1")() - err = Deploy(deployInput, mockV1Client, nil) - assert.NoError(t, err) - - defer testUtil.MockUserInput(t, "y")() - deployInput.RuntimeID = "test-id" - deployInput.Pytest = "pytest" - deployInput.Prompt = false - err = Deploy(deployInput, mockV1Client, nil) - assert.NoError(t, err) - - // test custom image - defer testUtil.MockUserInput(t, "y")() - deployInput.ImageName = "custom-image" - err = Deploy(deployInput, mockV1Client, nil) - assert.NoError(t, err) - - config.CFG.ProjectDeployment.SetProjectString("test-id") - // test both deploymentID and name used - defer testUtil.MockUserInput(t, "y")() - deployInput.DeploymentName = "test-name" - err = Deploy(deployInput, mockV1Client, nil) - assert.NoError(t, err) - - defer testUtil.MockUserInput(t, "y")() - deployInput.Pytest = "" - deployInput.WaitForStatus = true - sleepTime = 1 - err = Deploy(deployInput, mockV1Client, nil) - assert.ErrorIs(t, err, deployment.ErrTimedOut) - - mockV1Client.AssertExpectations(t) - mockImageHandler.AssertExpectations(t) - mockDAGCheck.AssertExpectations(t) - mockV1Client.AssertExpectations(t) -} - -func TestDeployOnRemoteExecutionDeployment(t *testing.T) { - mockV1Client := new(astrov1_mocks.ClientWithResponsesInterface) - deployInput := InputDeploy{ - Path: "./testfiles/", - RuntimeID: "", - WsID: ws, - Pytest: "parse", - EnvFile: "./testfiles/.env", - ImageName: "", - DeploymentName: "", - Prompt: true, - WaitForStatus: false, - Dags: false, - } - testUtil.InitTestConfig(testUtil.LocalPlatform) - config.CFG.ShowWarnings.SetHomeString("false") - - mockV1Client.On("GetDeploymentWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(&deploymentResponseRemoteExecution, nil).Times(6) - mockV1Client.On("ListDeploymentsWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(&mockListDeploymentsResponse, nil).Once() - mockV1Client.On("GetDeploymentOptionsWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(&getDeploymentOptionsResponse, nil).Times(5) - mockV1Client.On("CreateDeployWithResponse", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&createDeployResponse, nil).Times(5) - mockV1Client.On("FinalizeDeployWithResponse", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&finalizeDeployResponse, nil).Times(5) - - mockImageHandler := new(mocks.ImageHandler) - airflowImageHandler = func(image string) airflow.ImageHandler { - mockImageHandler.On("Build", mock.Anything, mock.Anything, mock.Anything).Return(nil) - mockImageHandler.On("Push", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return("", nil) - mockImageHandler.On("GetLabel", mock.Anything, runtimeImageLabel).Return("3.0-1", nil) - mockImageHandler.On("TagLocalImage", mock.Anything).Return(nil) - return mockImageHandler - } - - mockDAGCheck := new(mocks.DAGCheck) - dagCheckInit = func(airflowHome, envFile, dockerfile, imageName string) (airflow.DAGCheck, error) { - mockDAGCheck.On("Parse", mock.Anything, mock.Anything, mock.Anything).Return(nil) - mockDAGCheck.On("Pytest", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return("", nil) - return mockDAGCheck, nil - } - - ctx, err := config.GetCurrentContext() - assert.NoError(t, err) - ctx.Token = "test testing" - err = ctx.SetContext() - assert.NoError(t, err) - - // mock os.Stdin - input := []byte("1") - r, w, err := os.Pipe() - if err != nil { - t.Fatal(err) - } - _, err = w.Write(input) - if err != nil { - t.Error(err) - } - w.Close() - stdin := os.Stdin - // Restore stdin right after the test. - defer func() { os.Stdin = stdin }() - os.Stdin = r - - defer testUtil.MockUserInput(t, "1")() - err = Deploy(deployInput, mockV1Client, nil) - assert.NoError(t, err) - - defer testUtil.MockUserInput(t, "y")() - deployInput.RuntimeID = "test-id" - deployInput.Pytest = "pytest" - deployInput.Prompt = false - err = Deploy(deployInput, mockV1Client, nil) - assert.NoError(t, err) - - // test custom image - defer testUtil.MockUserInput(t, "y")() - deployInput.ImageName = "custom-image" - err = Deploy(deployInput, mockV1Client, nil) - assert.NoError(t, err) - - config.CFG.ProjectDeployment.SetProjectString("test-id") - // test both deploymentID and name used - defer testUtil.MockUserInput(t, "y")() - deployInput.DeploymentName = "test-name" - err = Deploy(deployInput, mockV1Client, nil) - assert.NoError(t, err) - - defer testUtil.MockUserInput(t, "y")() - deployInput.Pytest = "" - deployInput.WaitForStatus = true - sleepTime = 1 - err = Deploy(deployInput, mockV1Client, nil) - assert.ErrorIs(t, err, deployment.ErrTimedOut) - - mockV1Client.AssertExpectations(t) - mockImageHandler.AssertExpectations(t) - mockDAGCheck.AssertExpectations(t) - mockV1Client.AssertExpectations(t) -} - -func TestDeployOnCiCdEnforcedDeployment(t *testing.T) { - os.Mkdir("./testfiles/dags", os.ModePerm) - path := "./testfiles/dags/test.py" - fileutil.WriteStringToFile(path, "testing") - mockV1Client := new(astrov1_mocks.ClientWithResponsesInterface) - - deployInput := InputDeploy{ - Path: "./testfiles/", - RuntimeID: "", - WsID: ws, - Pytest: "parse", - EnvFile: "./testfiles/.env", - ImageName: "", - DeploymentName: "", - Prompt: true, - WaitForStatus: false, - Dags: false, - } - testUtil.InitTestConfig(testUtil.LocalPlatform) - config.CFG.ShowWarnings.SetHomeString("false") - canCiCdDeploy = func(astroAPIToken string) bool { - return false - } - - mockV1Client.On("ListDeploymentsWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(&mockListDeploymentsResponse, nil).Once() - mockV1Client.On("GetDeploymentWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(&deploymentResponseCICD, nil).Twice() - - defer testUtil.MockUserInput(t, "1")() - err := Deploy(deployInput, mockV1Client, nil) - assert.Contains(t, err.Error(), "cannot deploy since ci/cd enforcement is enabled for the deployment test-deployment. Please use API Tokens instead") - - defer os.RemoveAll("./testfiles/dags/") - - mockV1Client.AssertExpectations(t) -} - -func TestDeployWithDagsDeploySuccess(t *testing.T) { - os.Mkdir("./testfiles/dags", os.ModePerm) - path := "./testfiles/dags/test.py" - fileutil.WriteStringToFile(path, "testing") - mockV1Client := new(astrov1_mocks.ClientWithResponsesInterface) - - deployInput := InputDeploy{ - Path: "./testfiles/", - RuntimeID: "", - WsID: ws, - Pytest: "parse", - EnvFile: "./testfiles/.env", - ImageName: "", - DeploymentName: "", - Prompt: true, - WaitForStatus: false, - Dags: false, - } - testUtil.InitTestConfig(testUtil.LocalPlatform) - config.CFG.ShowWarnings.SetHomeString("false") - - mockV1Client.On("ListDeploymentsWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(&mockListDeploymentsResponse, nil).Times(2) - mockV1Client.On("GetDeploymentWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(&deploymentResponseDags, nil).Times(9) - mockV1Client.On("GetDeploymentOptionsWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(&getDeploymentOptionsResponse, nil).Times(7) - mockV1Client.On("CreateDeployWithResponse", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&createDeployResponse, nil).Times(7) - mockV1Client.On("FinalizeDeployWithResponse", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&finalizeDeployResponse, nil).Times(7) - - azureUploader = func(sasLink string, file io.Reader) (string, error) { - return "version-id", nil - } - - mockImageHandler := new(mocks.ImageHandler) - airflowImageHandler = func(image string) airflow.ImageHandler { - mockImageHandler.On("Build", mock.Anything, mock.Anything, mock.Anything).Return(nil) - mockImageHandler.On("Push", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return("", nil) - mockImageHandler.On("GetLabel", mock.Anything, runtimeImageLabel).Return("12.0.0", nil) - mockImageHandler.On("TagLocalImage", mock.Anything).Return(nil) - return mockImageHandler - } - - mockDAGCheck := new(mocks.DAGCheck) - dagCheckInit = func(airflowHome, envFile, dockerfile, imageName string) (airflow.DAGCheck, error) { - mockDAGCheck.On("Parse", mock.Anything, mock.Anything, mock.Anything).Return(nil) - mockDAGCheck.On("Pytest", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return("", nil) - return mockDAGCheck, nil - } - - ctx, err := config.GetCurrentContext() - assert.NoError(t, err) - ctx.Token = "test testing" - err = ctx.SetContext() - assert.NoError(t, err) - - // mock os.Stdin - input := []byte("1") - r, w, err := os.Pipe() - if err != nil { - t.Fatal(err) - } - _, err = w.Write(input) - if err != nil { - t.Error(err) - } - w.Close() - stdin := os.Stdin - // Restore stdin right after the test. - defer func() { os.Stdin = stdin }() - os.Stdin = r - - defer testUtil.MockUserInput(t, "1")() - err = Deploy(deployInput, mockV1Client, nil) - assert.NoError(t, err) - - defer testUtil.MockUserInput(t, "y")() - deployInput.RuntimeID = "test-id" - deployInput.Pytest = "pytest" - deployInput.Prompt = false - err = Deploy(deployInput, mockV1Client, nil) - assert.NoError(t, err) - - config.CFG.ProjectDeployment.SetProjectString("test-id") - // test both deploymentID and name used - defer testUtil.MockUserInput(t, "y")() - deployInput.DeploymentName = "test-name" - err = Deploy(deployInput, mockV1Client, nil) - assert.NoError(t, err) - - defer testUtil.MockUserInput(t, "y")() - deployInput.Pytest = "parse" - err = Deploy(deployInput, mockV1Client, nil) - assert.NoError(t, err) - - defer testUtil.MockUserInput(t, "y")() - deployInput.Pytest = parseAndPytest - err = Deploy(deployInput, mockV1Client, nil) - assert.NoError(t, err) - - // test custom image with dag deploy enabled - defer testUtil.MockUserInput(t, "y")() - deployInput.ImageName = "custom-image" - err = Deploy(deployInput, mockV1Client, nil) - assert.NoError(t, err) - - os.Mkdir("./testfiles1/", os.ModePerm) - fileutil.WriteStringToFile("./testfiles1/Dockerfile", "FROM quay.io/astronomer/astro-runtime:4.2.5") - fileutil.WriteStringToFile("./testfiles1/.env", "") - fileutil.WriteStringToFile("./testfiles1/.dockerignore", "") - - deployInput = InputDeploy{ - Path: "./testfiles1/", - RuntimeID: "", - WsID: ws, - Pytest: "parse", - EnvFile: "./testfiles/.env", - ImageName: "", - DeploymentName: "", - Prompt: true, - WaitForStatus: false, - Dags: false, - } - defer testUtil.MockUserInput(t, "1")() - err = Deploy(deployInput, mockV1Client, nil) - assert.NoError(t, err) - - defer os.RemoveAll("./testfiles1/") - defer os.RemoveAll("./testfiles/dags/") - - mockV1Client.AssertExpectations(t) - mockImageHandler.AssertExpectations(t) - mockDAGCheck.AssertExpectations(t) - mockV1Client.AssertExpectations(t) -} - -func TestDagsDeploySuccess(t *testing.T) { - mockV1Client := new(astrov1_mocks.ClientWithResponsesInterface) - - deployInput := InputDeploy{ - Path: "./testfiles/", - RuntimeID: "", - WsID: ws, - Pytest: "", - EnvFile: "", - ImageName: "", - DeploymentName: "", - Prompt: true, - Dags: true, - WaitForStatus: false, - DagsPath: "./testfiles/dags", - } - testUtil.InitTestConfig(testUtil.LocalPlatform) - config.CFG.ShowWarnings.SetHomeString("false") - mockV1Client.On("ListDeploymentsWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(&mockListDeploymentsResponse, nil).Times(6) - mockV1Client.On("GetDeploymentWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(&deploymentResponseDags, nil).Times(12) - mockV1Client.On("GetDeploymentOptionsWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(&getDeploymentOptionsResponse, nil).Times(4) - mockV1Client.On("CreateDeployWithResponse", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&createDeployResponse, nil).Times(6) - mockV1Client.On("FinalizeDeployWithResponse", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&finalizeDeployResponse, nil).Times(6) - - azureUploader = func(sasLink string, file io.Reader) (string, error) { - return "version-id", nil - } - - defer testUtil.MockUserInput(t, "1")() - err := Deploy(deployInput, mockV1Client, nil) - assert.NoError(t, err) - - // Test pytest with dags deploy - mockImageHandler := new(mocks.ImageHandler) - airflowImageHandler = func(image string) airflow.ImageHandler { - mockImageHandler.On("Build", mock.Anything, mock.Anything, mock.Anything).Return(nil) - mockImageHandler.On("Push", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return("", nil) - mockImageHandler.On("GetLabel", mock.Anything, runtimeImageLabel).Return("12.0.0", nil) - mockImageHandler.On("TagLocalImage", mock.Anything).Return(nil) - return mockImageHandler - } - - mockDAGCheck := new(mocks.DAGCheck) - dagCheckInit = func(airflowHome, envFile, dockerfile, imageName string) (airflow.DAGCheck, error) { - mockDAGCheck.On("Parse", mock.Anything, mock.Anything, mock.Anything).Return(nil) - mockDAGCheck.On("Pytest", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return("", nil) - return mockDAGCheck, nil - } - - defer testUtil.MockUserInput(t, "1")() - deployInput.Pytest = "parse" - err = Deploy(deployInput, mockV1Client, nil) - assert.NoError(t, err) - - defer testUtil.MockUserInput(t, "1")() - deployInput.Pytest = allTests - err = Deploy(deployInput, mockV1Client, nil) - assert.NoError(t, err) - - defer testUtil.MockUserInput(t, "1")() - deployInput.Pytest = parseAndPytest - err = Deploy(deployInput, mockV1Client, nil) - assert.NoError(t, err) - // image deploy - defer testUtil.MockUserInput(t, "1")() - deployInput.Image = true - - err = Deploy(deployInput, mockV1Client, nil) - assert.NoError(t, err) - - defer testUtil.MockUserInput(t, "1")() - deployInput.Pytest = "" - deployInput.WaitForStatus = true - dagOnlyDeploySleepTime = 1 - err = Deploy(deployInput, mockV1Client, nil) - assert.ErrorIs(t, err, deployment.ErrTimedOut) - - defer os.RemoveAll("./testfiles/dags/") - - mockV1Client.AssertExpectations(t) - mockV1Client.AssertExpectations(t) -} - -// A Deployment id names its own workspace. When the workspace in hand came -// from the current context rather than --workspace-id, the deploy follows the -// Deployment to its workspace instead of stopping. -func TestDagsDeployByIDFollowsTheDeploymentsWorkspace(t *testing.T) { - testUtil.InitTestConfig(testUtil.LocalPlatform) - config.CFG.ShowWarnings.SetHomeString("false") - mockV1Client := new(astrov1_mocks.ClientWithResponsesInterface) - mockV1Client.On("GetDeploymentWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(&deploymentResponseDags, nil) - mockV1Client.On("CreateDeployWithResponse", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&createDeployResponse, nil) - mockV1Client.On("FinalizeDeployWithResponse", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&finalizeDeployResponse, nil) - azureUploader = func(string, io.Reader) (string, error) { return "version-id", nil } - defer os.RemoveAll("./testfiles/dags/") - - deployInput := InputDeploy{ - Path: "./testfiles/", - RuntimeID: deploymentID, - WsID: "context-ws-id", - Dags: true, - DagsPath: "./testfiles/dags", - } - assert.NoError(t, Deploy(deployInput, mockV1Client, nil)) - mockV1Client.AssertNumberOfCalls(t, "CreateDeployWithResponse", 1) - - deployInput.WsIDFromFlag = true - err := Deploy(deployInput, mockV1Client, nil) - assert.EqualError(t, err, "deployment test-deployment-id is in workspace test-ws-id, not workspace context-ws-id given by --workspace. Pass --workspace test-ws-id, or leave the flag out") - mockV1Client.AssertNumberOfCalls(t, "CreateDeployWithResponse", 1) -} - -func TestImageOnlyDeploySuccess(t *testing.T) { - mockV1Client := new(astrov1_mocks.ClientWithResponsesInterface) - - deployInput := InputDeploy{ - Path: "./testfiles/", - RuntimeID: "", - WsID: ws, - Pytest: "", - EnvFile: "./testfiles/.env", - ImageName: "", - DeploymentName: "", - Prompt: true, - Dags: false, - Image: true, - WaitForStatus: false, - DagsPath: "./testfiles/dags", - } - testUtil.InitTestConfig(testUtil.LocalPlatform) - config.CFG.ShowWarnings.SetHomeString("false") - mockV1Client.On("ListDeploymentsWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(&mockListDeploymentsResponse, nil).Once() - mockV1Client.On("GetDeploymentWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(&deploymentResponseDags, nil).Times(2) - mockV1Client.On("GetDeploymentOptionsWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(&getDeploymentOptionsResponse, nil).Times(1) - mockV1Client.On("CreateDeployWithResponse", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&createDeployResponse, nil).Times(1) - mockV1Client.On("FinalizeDeployWithResponse", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&finalizeDeployResponse, nil).Times(1) - - // Test pytest with dags deploy - mockImageHandler := new(mocks.ImageHandler) - airflowImageHandler = func(image string) airflow.ImageHandler { - mockImageHandler.On("Build", mock.Anything, mock.Anything, mock.Anything).Return(nil) - mockImageHandler.On("Push", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return("", nil) - mockImageHandler.On("GetLabel", mock.Anything, runtimeImageLabel).Return("12.0.0", nil) - mockImageHandler.On("TagLocalImage", mock.Anything).Return(nil) - return mockImageHandler - } - - mockDAGCheck := new(mocks.DAGCheck) - dagCheckInit = func(airflowHome, envFile, dockerfile, imageName string) (airflow.DAGCheck, error) { - mockDAGCheck.On("Parse", mock.Anything, mock.Anything, mock.Anything).Return(nil) - mockDAGCheck.On("Pytest", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return("", nil) - return mockDAGCheck, nil - } - - defer testUtil.MockUserInput(t, "1")() - err := Deploy(deployInput, mockV1Client, nil) - assert.NoError(t, err) - - defer os.RemoveAll("./testfiles/dags/") - - mockV1Client.AssertExpectations(t) - mockV1Client.AssertExpectations(t) -} - -func TestNoDagsDeploy(t *testing.T) { - testUtil.InitTestConfig(testUtil.LocalPlatform) - config.CFG.ShowWarnings.SetHomeString("true") - mockV1Client := new(astrov1_mocks.ClientWithResponsesInterface) - - ctx, err := config.GetCurrentContext() - assert.NoError(t, err) - ctx.Token = "test testing" - err = ctx.SetContext() - assert.NoError(t, err) - - mockV1Client.On("ListDeploymentsWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(&mockListDeploymentsResponse, nil).Times(1) - mockV1Client.On("GetDeploymentWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(&deploymentResponseDags, nil).Times(2) - mockV1Client.On("CreateDeployWithResponse", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&createDeployResponse, nil).Times(1) - - deployInput := InputDeploy{ - Path: "./testfiles/", - RuntimeID: "", - WsID: ws, - Pytest: "", - EnvFile: "", - ImageName: "", - DeploymentName: "", - Prompt: true, - WaitForStatus: false, - Dags: true, - } - defer testUtil.MockUserInput(t, "1")() - err = Deploy(deployInput, mockV1Client, nil) - assert.NoError(t, err) - - mockV1Client.AssertExpectations(t) - mockV1Client.AssertExpectations(t) -} - -func TestNoDagsDeployForceSkipsPrompt(t *testing.T) { - testUtil.InitTestConfig(testUtil.LocalPlatform) - config.CFG.ShowWarnings.SetHomeString("true") - mockV1Client := new(astrov1_mocks.ClientWithResponsesInterface) - - ctx, err := config.GetCurrentContext() - assert.NoError(t, err) - ctx.Token = "test testing" - err = ctx.SetContext() - assert.NoError(t, err) - - mockV1Client.On("GetDeploymentWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(&deploymentResponseDags, nil) - mockV1Client.On("CreateDeployWithResponse", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&createDeployResponse, nil) - mockV1Client.On("FinalizeDeployWithResponse", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&finalizeDeployResponse, nil) - - azureUploader = func(sasLink string, file io.Reader) (string, error) { - return "version-id", nil - } - - deployInput := InputDeploy{ - Path: "./testfiles/", - RuntimeID: deploymentID, - WsID: ws, - Dags: true, - Force: true, - } - err = Deploy(deployInput, mockV1Client, nil) - assert.NoError(t, err) - - defer os.RemoveAll("./testfiles/dags/") - - mockV1Client.AssertExpectations(t) - mockV1Client.AssertExpectations(t) -} - -func TestNoDagsImageDeployForceSkipsPrompt(t *testing.T) { - testUtil.InitTestConfig(testUtil.LocalPlatform) - config.CFG.ShowWarnings.SetHomeString("true") - mockV1Client := new(astrov1_mocks.ClientWithResponsesInterface) - - ctx, err := config.GetCurrentContext() - assert.NoError(t, err) - ctx.Token = "test testing" - err = ctx.SetContext() - assert.NoError(t, err) - - mockV1Client.On("GetDeploymentWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(&deploymentResponseDags, nil) - mockV1Client.On("GetDeploymentOptionsWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(&getDeploymentOptionsResponse, nil) - mockV1Client.On("CreateDeployWithResponse", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&createDeployResponse, nil) - mockV1Client.On("FinalizeDeployWithResponse", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&finalizeDeployResponse, nil) - - azureUploader = func(sasLink string, file io.Reader) (string, error) { - return "version-id", nil - } - - mockImageHandler := new(mocks.ImageHandler) - airflowImageHandler = func(image string) airflow.ImageHandler { - mockImageHandler.On("Build", mock.Anything, mock.Anything, mock.Anything).Return(nil) - mockImageHandler.On("Push", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return("", nil) - mockImageHandler.On("GetLabel", mock.Anything, runtimeImageLabel).Return("12.0.0", nil) - mockImageHandler.On("TagLocalImage", mock.Anything).Return(nil) - return mockImageHandler - } - - deployInput := InputDeploy{ - Path: "./testfiles/", - RuntimeID: deploymentID, - WsID: ws, - EnvFile: "./testfiles/.env", - Dags: false, - Force: true, - } - err = Deploy(deployInput, mockV1Client, nil) - assert.NoError(t, err) - - mockV1Client.AssertExpectations(t) - mockV1Client.AssertExpectations(t) -} - -// Regression test for v1.42.0: --image-name on a Remote Execution -// deployment auto-set Image=true and tripped the dagDeployEnabled guard, -// returning a bogus "Dag-only deploys are not enabled" error even though -// RE deployments don't deploy DAGs to the orchestration plane. -func TestImageDeployOnRemoteExecutionDeploymentSucceedsWithoutDagDeploy(t *testing.T) { - testUtil.InitTestConfig(testUtil.LocalPlatform) - config.CFG.ShowWarnings.SetHomeString("false") - - mockV1Client := new(astrov1_mocks.ClientWithResponsesInterface) - - mockV1Client.On("GetDeploymentWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(&deploymentResponseRemoteExecution, nil) - mockV1Client.On("GetDeploymentOptionsWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(&getDeploymentOptionsResponse, nil) - mockV1Client.On("CreateDeployWithResponse", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&createDeployResponse, nil) - mockV1Client.On("FinalizeDeployWithResponse", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&finalizeDeployResponse, nil) - - mockImageHandler := new(mocks.ImageHandler) - airflowImageHandler = func(image string) airflow.ImageHandler { - mockImageHandler.On("Build", mock.Anything, mock.Anything, mock.Anything).Return(nil) - mockImageHandler.On("Push", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return("", nil) - mockImageHandler.On("GetLabel", mock.Anything, runtimeImageLabel).Return("3.0-1", nil) - mockImageHandler.On("TagLocalImage", mock.Anything).Return(nil) - return mockImageHandler - } - - ctx, err := config.GetCurrentContext() - assert.NoError(t, err) - ctx.Token = "test testing" - err = ctx.SetContext() - assert.NoError(t, err) - - deployInput := InputDeploy{ - Path: "./testfiles/", - RuntimeID: deploymentID, - WsID: ws, - EnvFile: "./testfiles/.env", - ImageName: "custom-image", - Image: true, - Force: true, - } - - err = Deploy(deployInput, mockV1Client, nil) - assert.NoError(t, err) - if err != nil { - assert.NotContains(t, err.Error(), "Dag-only deploys are not enabled") - } - - mockV1Client.AssertExpectations(t) -} - -func TestDagsDeployFailed(t *testing.T) { - testUtil.InitTestConfig(testUtil.LocalPlatform) - config.CFG.ShowWarnings.SetHomeString("false") - mockV1Client := new(astrov1_mocks.ClientWithResponsesInterface) - - deployInput := InputDeploy{ - Path: "./testfiles/", - RuntimeID: deploymentID, - WsID: ws, - Pytest: "", - EnvFile: "", - ImageName: "", - DeploymentName: "", - Prompt: true, - WaitForStatus: false, - Dags: true, - } - mockV1Client.On("ListDeploymentsWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(&mockListDeploymentsResponse, nil).Times(3) - mockV1Client.On("GetDeploymentWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(&deploymentResponse, nil).Times(6) - mockV1Client.On("GetDeploymentOptionsWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(&getDeploymentOptionsResponse, nil).Times(2) - mockV1Client.On("CreateDeployWithResponse", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&createDeployResponse, nil).Times(3) - - defer testUtil.MockUserInput(t, "y")() - err := Deploy(deployInput, mockV1Client, nil) - assert.Equal(t, err.Error(), "Dag-only deploys are not enabled for this Deployment. Run 'astro deployment update test-deployment-id --dag-deploy enable' to enable Dag-only deploys") - - mockImageHandler := new(mocks.ImageHandler) - airflowImageHandler = func(image string) airflow.ImageHandler { - mockImageHandler.On("Build", mock.Anything, mock.Anything, mock.Anything).Return(nil) - mockImageHandler.On("GetLabel", mock.Anything, runtimeImageLabel).Return("12.0.0", nil) - return mockImageHandler - } - - mockDAGCheck := new(mocks.DAGCheck) - dagCheckInit = func(airflowHome, envFile, dockerfile, imageName string) (airflow.DAGCheck, error) { - mockDAGCheck.On("Parse", mock.Anything, mock.Anything, mock.Anything).Return(errMock) - mockDAGCheck.On("Pytest", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return("", errMock) - return mockDAGCheck, nil - } - - defer testUtil.MockUserInput(t, "y")() - deployInput.Pytest = "parse" - err = Deploy(deployInput, mockV1Client, nil) - assert.Error(t, err) - - defer testUtil.MockUserInput(t, "y")() - deployInput.Pytest = allTests - err = Deploy(deployInput, mockV1Client, nil) - assert.Error(t, err) - - mockV1Client.AssertExpectations(t) - mockV1Client.AssertExpectations(t) -} - -func TestDeployFailure(t *testing.T) { - os.Mkdir("./testfiles/dags", os.ModePerm) - path := "./testfiles/dags/test.py" - fileutil.WriteStringToFile(path, "testing") - - defer os.RemoveAll("./testfiles/dags/") - - // no context set failure - testUtil.InitTestConfig(testUtil.LocalPlatform) - err := config.ResetCurrentContext() - assert.NoError(t, err) - mockV1Client := new(astrov1_mocks.ClientWithResponsesInterface) - - deployInput := InputDeploy{ - Path: "./testfiles/", - RuntimeID: deploymentID, - WsID: ws, - Pytest: "parse", - EnvFile: "./testfiles/.env", - ImageName: "", - DeploymentName: "", - Prompt: true, - WaitForStatus: false, - Dags: false, - } - - defer testUtil.MockUserInput(t, "y")() - err = Deploy(deployInput, mockV1Client, nil) - assert.EqualError(t, err, "no context set, have you authenticated to Astro or APC? Run astro login and try again") - - testUtil.InitTestConfig(testUtil.CloudPlatform) - mockV1Client.On("ListDeploymentsWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(&mockListDeploymentsResponse, nil).Times(3) - mockV1Client.On("GetDeploymentWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(&deploymentResponse, nil).Times(6) - mockV1Client.On("GetDeploymentOptionsWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(&getDeploymentOptionsResponse, nil).Times(1) - mockV1Client.On("CreateDeployWithResponse", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&createDeployResponse, nil).Times(2) - - mockImageHandler := new(mocks.ImageHandler) - airflowImageHandler = func(image string) airflow.ImageHandler { - mockImageHandler.On("Build", mock.Anything, mock.Anything, mock.Anything).Return(nil) - mockImageHandler.On("GetLabel", mock.Anything, runtimeImageLabel).Return("12.0.0", nil) - return mockImageHandler - } - - mockDAGCheck := new(mocks.DAGCheck) - dagCheckInit = func(airflowHome, envFile, dockerfile, imageName string) (airflow.DAGCheck, error) { - mockDAGCheck.On("Parse", mock.Anything, mock.Anything, mock.Anything).Return(errMock) - return mockDAGCheck, nil - } - - // mock os.Stdin - input := []byte("1") - r, w, err := os.Pipe() - if err != nil { - t.Fatal(err) - } - _, err = w.Write(input) - if err != nil { - t.Error(err) - } - w.Close() - stdin := os.Stdin - // Restore stdin right after the test. - defer func() { os.Stdin = stdin }() - os.Stdin = r - - defer testUtil.MockUserInput(t, "1")() - deployInput.RuntimeID = "" - err = Deploy(deployInput, mockV1Client, nil) - assert.ErrorIs(t, err, errDagsParseFailed) - - defer testUtil.MockUserInput(t, "y")() - deployInput.RuntimeID = deploymentID - deployInput.WsID = "invalid-workspace" - deployInput.WsIDFromFlag = true - err = Deploy(deployInput, mockV1Client, nil) - assert.EqualError(t, err, "deployment test-deployment-id is in workspace test-ws-id, not workspace invalid-workspace given by --workspace. Pass --workspace test-ws-id, or leave the flag out") - deployInput.WsIDFromFlag = false - - defer testUtil.MockUserInput(t, "y")() - deployInput.WsID = ws - deployInput.EnvFile = "invalid-path" - err = Deploy(deployInput, mockV1Client, nil) - assert.ErrorIs(t, err, envFileMissing) - - mockV1Client.AssertExpectations(t) - mockImageHandler.AssertExpectations(t) - - mockDAGCheck.AssertExpectations(t) - mockV1Client.AssertExpectations(t) -} - -func TestDeployMonitoringDAGNonHosted(t *testing.T) { - deployInput := InputDeploy{ - Path: "./testfiles/", - RuntimeID: deploymentID, - WsID: ws, - Pytest: "", - EnvFile: "", - ImageName: "", - DeploymentName: "", - Prompt: true, - Dags: true, - DagsPath: "./testfiles/dags", - } - testUtil.InitTestConfig(testUtil.LocalPlatform) - config.CFG.ShowWarnings.SetHomeString("false") - mockV1Client := new(astrov1_mocks.ClientWithResponsesInterface) - - ctx, err := config.GetCurrentContext() - assert.NoError(t, err) - ctx.OrganizationProduct = "HYBRID" - err = ctx.SetContext() - assert.NoError(t, err) - - mockV1Client.On("GetDeploymentOptionsWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(&getDeploymentOptionsResponse, nil).Times(3) - mockV1Client.On("ListDeploymentsWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(&mockListDeploymentsResponse, nil).Times(4) - mockV1Client.On("GetDeploymentWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(&deploymentResponseDags, nil).Times(8) - mockV1Client.On("CreateDeployWithResponse", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&createDeployResponse, nil).Times(4) - mockV1Client.On("FinalizeDeployWithResponse", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&finalizeDeployResponse, nil).Times(4) - - azureUploader = func(sasLink string, file io.Reader) (string, error) { - _, err = os.Stat("./testfiles/dags/astronomer_monitoring_dag.py") - assert.NoError(t, err) - return "version-id", nil - } - - defer testUtil.MockUserInput(t, "y")() - err = Deploy(deployInput, mockV1Client, nil) - assert.NoError(t, err) - - // Test pytest with dags deploy - mockImageHandler := new(mocks.ImageHandler) - airflowImageHandler = func(image string) airflow.ImageHandler { - mockImageHandler.On("Build", mock.Anything, mock.Anything, mock.Anything).Return(nil) - mockImageHandler.On("Push", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) - mockImageHandler.On("GetLabel", mock.Anything, runtimeImageLabel).Return("12.0.0", nil) - mockImageHandler.On("TagLocalImage", mock.Anything).Return(nil) - return mockImageHandler - } - - mockDAGCheck := new(mocks.DAGCheck) - dagCheckInit = func(airflowHome, envFile, dockerfile, imageName string) (airflow.DAGCheck, error) { - mockDAGCheck.On("Parse", mock.Anything, mock.Anything, mock.Anything).Return(nil) - mockDAGCheck.On("Pytest", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return("", nil) - return mockDAGCheck, nil - } - - defer testUtil.MockUserInput(t, "y")() - deployInput.Pytest = "parse" - err = Deploy(deployInput, mockV1Client, nil) - assert.NoError(t, err) - - defer testUtil.MockUserInput(t, "y")() - deployInput.Pytest = allTests - err = Deploy(deployInput, mockV1Client, nil) - assert.NoError(t, err) - - defer testUtil.MockUserInput(t, "y")() - deployInput.Pytest = parseAndPytest - err = Deploy(deployInput, mockV1Client, nil) - assert.NoError(t, err) - - defer os.RemoveAll("./testfiles/dags/") - - mockV1Client.AssertExpectations(t) - mockV1Client.AssertExpectations(t) -} - -func TestDeployNoMonitoringDAGHosted(t *testing.T) { - dagsDir := "./testfiles/dags" - err := os.MkdirAll(dagsDir, os.ModePerm) - assert.NoError(t, err) - defer os.RemoveAll(dagsDir) - - deployInput := InputDeploy{ - Path: "./testfiles/", - RuntimeID: deploymentID, - WsID: ws, - Pytest: "", - EnvFile: "", - ImageName: "", - DeploymentName: "", - Prompt: true, - Dags: true, - DagsPath: "./testfiles/dags", - } - testUtil.InitTestConfig(testUtil.LocalPlatform) - config.CFG.ShowWarnings.SetHomeString("false") - mockV1Client := new(astrov1_mocks.ClientWithResponsesInterface) - - ctx, err := config.GetCurrentContext() - assert.NoError(t, err) - ctx.OrganizationProduct = "HOSTED" - err = ctx.SetContext() - assert.NoError(t, err) - - mockV1Client.On("GetDeploymentOptionsWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(&getDeploymentOptionsResponse, nil).Times(3) - mockV1Client.On("ListDeploymentsWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(&mockListDeploymentsResponse, nil).Times(4) - mockV1Client.On("GetDeploymentWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(&deploymentResponseDags, nil).Times(8) - mockV1Client.On("CreateDeployWithResponse", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&createDeployResponse, nil).Times(4) - mockV1Client.On("FinalizeDeployWithResponse", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&finalizeDeployResponse, nil).Times(4) - - azureUploader = func(sasLink string, file io.Reader) (string, error) { - _, err = os.Stat("./testfiles/dags/astronomer_monitoring_dag.py") - assert.ErrorIs(t, err, os.ErrNotExist) - return "version-id", nil - } - - defer testUtil.MockUserInput(t, "y")() - err = Deploy(deployInput, mockV1Client, nil) - assert.NoError(t, err) - - // Test pytest with dags deploy - mockImageHandler := new(mocks.ImageHandler) - airflowImageHandler = func(image string) airflow.ImageHandler { - mockImageHandler.On("Build", mock.Anything, mock.Anything, mock.Anything).Return(nil) - mockImageHandler.On("Push", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) - mockImageHandler.On("GetLabel", mock.Anything, runtimeImageLabel).Return("12.0.0", nil) - mockImageHandler.On("TagLocalImage", mock.Anything).Return("", nil) - return mockImageHandler - } - - mockDAGCheck := new(mocks.DAGCheck) - dagCheckInit = func(airflowHome, envFile, dockerfile, imageName string) (airflow.DAGCheck, error) { - mockDAGCheck.On("Parse", mock.Anything, mock.Anything, mock.Anything).Return(nil) - mockDAGCheck.On("Pytest", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return("", nil) - return mockDAGCheck, nil - } - - defer testUtil.MockUserInput(t, "y")() - deployInput.Pytest = "parse" - err = Deploy(deployInput, mockV1Client, nil) - assert.NoError(t, err) - - defer testUtil.MockUserInput(t, "y")() - deployInput.Pytest = allTests - err = Deploy(deployInput, mockV1Client, nil) - assert.NoError(t, err) - - defer testUtil.MockUserInput(t, "y")() - deployInput.Pytest = parseAndPytest - err = Deploy(deployInput, mockV1Client, nil) - assert.NoError(t, err) - - defer os.RemoveAll("./testfiles/dags/") - - mockV1Client.AssertExpectations(t) - mockV1Client.AssertExpectations(t) -} - -func TestBuildImageFailure(t *testing.T) { - testUtil.InitTestConfig(testUtil.LocalPlatform) - - mockImageHandler := new(mocks.ImageHandler) - mockV1Client := new(astrov1_mocks.ClientWithResponsesInterface) - - // image build failure - airflowImageHandler = func(image string) airflow.ImageHandler { - mockImageHandler.On("Build", mock.Anything, mock.Anything, mock.Anything).Return(errMock).Once() - return mockImageHandler - } - _, err := buildImage("./testfiles/", "4.2.5", "", "", "", nil, false, false, mockV1Client) - assert.ErrorIs(t, err, errMock) - - // dockerfile parsing error: no imageName, version label empty → enters parse branch - airflowImageHandler = func(image string) airflow.ImageHandler { - mockImageHandler.On("Build", mock.Anything, mock.Anything, mock.Anything).Return(nil).Once() - mockImageHandler.On("GetLabel", mock.Anything, runtimeImageLabel).Return("", nil).Once() - return mockImageHandler - } - dockerfile = "Dockerfile.invalid" - _, err = buildImage("./testfiles/", "4.2.5", "", "", "", nil, false, false, mockV1Client) - assert.Error(t, err) - assert.Contains(t, err.Error(), "failed to parse dockerfile") - - // --image-name provided: Dockerfile parse is skipped, so a missing/invalid - // Dockerfile must not cause an error - airflowImageHandler = func(image string) airflow.ImageHandler { - mockImageHandler.On("TagLocalImage", mock.Anything).Return(nil).Once() - mockImageHandler.On("GetLabel", mock.Anything, runtimeImageLabel).Return("12.0.0", nil).Once() - return mockImageHandler - } - mockV1Client.On("GetDeploymentOptionsWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(&getDeploymentOptionsResponse, nil).Once() - dockerfile = "Dockerfile.invalid" // would normally cause a parse error - _, err = buildImage("./testfiles/", "4.2.5", "", "my-registry/my-image:tag", "", nil, false, false, mockV1Client) - assert.NoError(t, err, "Dockerfile should not be parsed when --image-name is provided") - - // failed to get runtime releases - airflowImageHandler = func(image string) airflow.ImageHandler { - mockImageHandler.On("Build", mock.Anything, mock.Anything, mock.Anything).Return(nil).Once() - mockImageHandler.On("GetLabel", mock.Anything, runtimeImageLabel).Return("12.0.0", nil).Once() - return mockImageHandler - } - dockerfile = "Dockerfile" - mockV1Client.On("GetDeploymentOptionsWithResponse", mock.Anything, mock.Anything, mock.Anything).Return(&getDeploymentOptionsResponse, errMock).Once() - _, err = buildImage("./testfiles/", "4.2.5", "", "", "", nil, false, false, mockV1Client) - assert.ErrorIs(t, err, errMock) - mockV1Client.AssertExpectations(t) - mockV1Client.AssertExpectations(t) - mockImageHandler.AssertExpectations(t) -} - -func TestValidRuntimeVersion(t *testing.T) { - testCases := []struct { - name string - currentVersion string - newVersion string - deploymentOptions []string - expected bool - expectedError string - }{ - // Empty current version cases - { - name: "empty current version allows any upgrade", - currentVersion: "", - newVersion: "4.2.6", - deploymentOptions: []string{"4.2.6"}, - expected: true, - }, - - // AF2 version upgrade cases - { - name: "newer AF2 version is valid upgrade", - currentVersion: "4.2.5", - newVersion: "4.2.6", - deploymentOptions: []string{"4.2.6"}, - expected: true, - }, - { - name: "older AF2 version is invalid upgrade", - currentVersion: "4.2.6", - newVersion: "4.2.5", - deploymentOptions: []string{"4.2.5"}, - expected: false, - expectedError: "Cannot deploy a downgraded Astro Runtime version", - }, - - // AF2 to AF3 upgrade cases - { - name: "AF2 >= 12.0.0 to AF3 version is valid upgrade", - currentVersion: "12.0.0", - newVersion: "3.0-1", - deploymentOptions: []string{"3.0-1"}, - expected: true, - }, - { - name: "AF2 < 12.0.0 to AF3 version is invalid upgrade", - currentVersion: "4.2.5", - newVersion: "3.0-1", - deploymentOptions: []string{"3.0-1"}, - expected: false, - expectedError: "Can only upgrade deployment from Airflow 2 to Airflow 3 with deployment at Astro Runtime 12.0.0 or higher", - }, - - // AF3 version cases - { - name: "newer AF3 version is valid upgrade", - currentVersion: "3.0-1", - newVersion: "3.1-1", - deploymentOptions: []string{"3.1-1"}, - expected: true, - }, - { - name: "older AF3 version is invalid upgrade", - currentVersion: "3.1-1", - newVersion: "3.0-3", - deploymentOptions: []string{"3.0-3"}, - expected: false, - expectedError: "Cannot deploy a downgraded Astro Runtime version", - }, - { - name: "AF3 to AF2 is invalid upgrade", - currentVersion: "3.0-1", - newVersion: "4.2.6", - deploymentOptions: []string{"4.2.6"}, - expected: false, - expectedError: "Cannot deploy a downgraded Astro Runtime version", - }, - - // Deployment options validation cases - { - name: "version not in deployment options is invalid", - currentVersion: "4.2.5", - newVersion: "4.2.6", - deploymentOptions: []string{"4.2.5", "4.2.4"}, - expected: false, - expectedError: "Cannot deploy an unsupported Astro Runtime version", - }, - { - name: "version in deployment options is valid", - currentVersion: "4.2.5", - newVersion: "4.2.6", - deploymentOptions: []string{"4.2.6", "4.2.5"}, - expected: true, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - // Capture stdout to check error messages - oldStdout := os.Stdout - r, w, _ := os.Pipe() - os.Stdout = w - outC := make(chan string) - go func() { - var buf bytes.Buffer - io.Copy(&buf, r) - outC <- buf.String() - }() - - result := ValidRuntimeVersion(tc.currentVersion, tc.newVersion, tc.deploymentOptions) - - // Restore stdout - w.Close() - os.Stdout = oldStdout - - // Check result - assert.Equal(t, tc.expected, result) - - // Check error message if expected - if tc.expectedError != "" { - output := <-outC - assert.Contains(t, output, tc.expectedError) - } - }) - } -} - -// mockTransport implements http.RoundTripper for mocking HTTP responses -type mockTransport struct { - response *http.Response -} - -func (m *mockTransport) RoundTrip(*http.Request) (*http.Response, error) { - return m.response, nil -} - -func TestWarnNonLatestVersion(t *testing.T) { - // Create a mock HTTP client - mockClient := &httputil.HTTPClient{ - HTTPClient: &http.Client{ - Transport: &mockTransport{ - response: &http.Response{ - StatusCode: 200, - Body: io.NopCloser(strings.NewReader(`{ - "runtimeVersions": { - "5.0.1": { - "metadata": { - "airflowVersion": "2.2.5", - "channel": "stable", - "releaseDate": "2024-01-01" - }, - "migrations": { - "airflowDatabase": false - } - }, - "5.0.2": { - "metadata": { - "airflowVersion": "2.2.6", - "channel": "stable", - "releaseDate": "2024-01-02" - }, - "migrations": { - "airflowDatabase": false - } - } - } - }`)), - }, - }, - }, - } - - t.Run("older version shows warning", func(t *testing.T) { - orgStdout := os.Stdout - defer func() { os.Stdout = orgStdout }() - - r, w, _ := os.Pipe() - os.Stdout = w - - WarnIfNonLatestVersion("4.2.5", mockClient) - - w.Close() - out, _ := io.ReadAll(r) - assert.Equal(t, "WARNING! You are currently running Astro Runtime Version 4.2.5\nConsider upgrading to the latest version, Astro Runtime 5.0.2\n", string(out)) - }) - - t.Run("latest version shows no warning", func(t *testing.T) { - orgStdout := os.Stdout - defer func() { os.Stdout = orgStdout }() - - r, w, _ := os.Pipe() - os.Stdout = w - - WarnIfNonLatestVersion("5.0.2", mockClient) - - w.Close() - out, _ := io.ReadAll(r) - assert.Equal(t, "", string(out)) - }) - - t.Run("newer version shows no warning", func(t *testing.T) { - orgStdout := os.Stdout - defer func() { os.Stdout = orgStdout }() - - r, w, _ := os.Pipe() - os.Stdout = w - - WarnIfNonLatestVersion("5.0.3", mockClient) - - w.Close() - out, _ := io.ReadAll(r) - assert.Equal(t, "", string(out)) - }) -} - -func TestCheckPyTest(t *testing.T) { - mockDeployImage := "test-image" - - mockDAGCheck := new(mocks.DAGCheck) - mockDAGCheck.On("Pytest", "", "", mockDeployImage, "", mock.Anything).Return("", errMock).Once() - - // random error on running airflow pytest - err := checkPytest("", mockDeployImage, nil, mockDAGCheck) - assert.ErrorIs(t, err, errMock) - mockDAGCheck.AssertExpectations(t) - - // airflow pytest exited with status code 1 - mockDAGCheck.On("Pytest", "", "", mockDeployImage, "", mock.Anything).Return("exit code 1", errMock).Once() - err = checkPytest("", mockDeployImage, nil, mockDAGCheck) - assert.Error(t, err) - assert.Contains(t, err.Error(), "at least 1 pytest in your tests directory failed. Fix the issues listed or rerun the command without the '--pytest' flag to deploy") - mockDAGCheck.AssertExpectations(t) -} - func TestDeployClientImage(t *testing.T) { // Store original DockerLogin function to restore after tests originalDockerLogin := airflow.DockerLogin @@ -2494,258 +1206,3 @@ func TestDeployDagsBundleLayout(t *testing.T) { }) } } - -func TestBuildImageWithoutDagsPreservesDockerignore(t *testing.T) { - tests := []struct { - name string - original string - }{ - { - name: "LF file with trailing newline is restored exactly", - original: "some/path/\nother/\n", - }, - { - name: "LF file without trailing newline is restored exactly", - original: "some/path/\nother/", - }, - { - name: "CRLF file is restored byte-for-byte (Windows)", - original: "some/path/\r\nother/\r\n", - }, - { - name: "file that already contains dags/ is restored exactly", - original: "some/path/\ndags/\nother/\n", - }, - { - name: "empty file is restored as empty", - original: "", - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - dir := t.TempDir() - dockerignorePath := filepath.Join(dir, ".dockerignore") - err := os.WriteFile(dockerignorePath, []byte(tc.original), 0o644) - assert.NoError(t, err) - - mockImageHandler := new(mocks.ImageHandler) - mockImageHandler.On("Build", mock.Anything, mock.Anything, mock.Anything).Return(nil) - - err = buildImageWithoutDags(dir, nil, mockImageHandler) - assert.NoError(t, err) - - got, err := os.ReadFile(dockerignorePath) - assert.NoError(t, err) - assert.Equal(t, tc.original, string(got)) - - mockImageHandler.AssertExpectations(t) - }) - } -} - -func TestBuildImageWithoutDagsCleansUpCreatedDockerignore(t *testing.T) { - dir := t.TempDir() - dockerignorePath := filepath.Join(dir, ".dockerignore") - - mockImageHandler := new(mocks.ImageHandler) - mockImageHandler.On("Build", mock.Anything, mock.Anything, mock.Anything).Return(nil) - - err := buildImageWithoutDags(dir, nil, mockImageHandler) - assert.NoError(t, err) - - _, statErr := os.Stat(dockerignorePath) - assert.True(t, os.IsNotExist(statErr), "expected .dockerignore not to exist, got: %v", statErr) - - mockImageHandler.AssertExpectations(t) -} - -func TestBuildImageWithoutDagsAppendsDagsDuringBuild(t *testing.T) { - tests := []struct { - name string - original string - expectedDuringBuild string - }{ - { - name: "appends dags/ when missing and file has trailing newline", - original: "some/path/\n", - expectedDuringBuild: "some/path/\ndags/\n", - }, - { - name: "appends dags/ when missing and file has no trailing newline", - original: "some/path/", - expectedDuringBuild: "some/path/\ndags/\n", - }, - { - name: "leaves file alone when dags/ already present", - original: "some/path/\ndags/\n", - expectedDuringBuild: "some/path/\ndags/\n", - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - dir := t.TempDir() - dockerignorePath := filepath.Join(dir, ".dockerignore") - err := os.WriteFile(dockerignorePath, []byte(tc.original), 0o644) - assert.NoError(t, err) - - mockImageHandler := new(mocks.ImageHandler) - mockImageHandler.On("Build", mock.Anything, mock.Anything, mock.Anything).Run(func(_ mock.Arguments) { - got, readErr := os.ReadFile(dockerignorePath) - assert.NoError(t, readErr) - assert.Equal(t, tc.expectedDuringBuild, string(got)) - }).Return(nil) - - err = buildImageWithoutDags(dir, nil, mockImageHandler) - assert.NoError(t, err) - - mockImageHandler.AssertExpectations(t) - }) - } -} - -func TestBuildImageWithoutDagsSurfacesRestoreError(t *testing.T) { - dir := t.TempDir() - dockerignorePath := filepath.Join(dir, ".dockerignore") - assert.NoError(t, os.WriteFile(dockerignorePath, []byte("some/path/\n"), 0o644)) - - mockImageHandler := new(mocks.ImageHandler) - // Sabotage the restore: swap the .dockerignore file for a directory during - // the build so the deferred WriteFile cannot put the original bytes back. - mockImageHandler.On("Build", mock.Anything, mock.Anything, mock.Anything).Run(func(_ mock.Arguments) { - assert.NoError(t, os.Remove(dockerignorePath)) - assert.NoError(t, os.Mkdir(dockerignorePath, 0o755)) - }).Return(nil) - - err := buildImageWithoutDags(dir, nil, mockImageHandler) - assert.Error(t, err) - assert.Contains(t, err.Error(), "restoring") - - mockImageHandler.AssertExpectations(t) -} - -func TestRemoveDagsFromDockerIgnore(t *testing.T) { - tests := []struct { - name string - input string - expected string - }{ - { - name: "file with trailing newline preserves it", - input: "some/path/\ndags/\nother/\n", - expected: "some/path/\nother/\n", - }, - { - name: "file without trailing newline preserves that", - input: "some/path/\ndags/", - expected: "some/path/", - }, - { - name: "dags as last line with trailing newline", - input: "some/path/\ndags/\n", - expected: "some/path/\n", - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, ".dockerignore") - err := os.WriteFile(path, []byte(tc.input), 0o644) - assert.NoError(t, err) - - err = removeDagsFromDockerIgnore(path) - assert.NoError(t, err) - - got, err := os.ReadFile(path) - assert.NoError(t, err) - assert.Equal(t, tc.expected, string(got)) - }) - } -} - -func TestCreateDeployWithDagBundle(t *testing.T) { - testUtil.InitTestConfig(testUtil.LocalPlatform) - - t.Run("maps a plain deploy to the IMAGE type", func(t *testing.T) { - mockV1Alpha1Client := new(astrov1alpha1_mocks.ClientWithResponsesInterface) - mockV1Alpha1Client.On("CreateDeployWithResponse", mock.Anything, "org-id", "deployment-id", mock.MatchedBy(func(req astrov1alpha1.CreateDeployRequest) bool { - return req.Type == astrov1alpha1.CreateDeployRequestTypeIMAGE && - req.DagBundleName != nil && *req.DagBundleName == "finance" - })).Return(&astrov1alpha1.CreateDeployResponse{ - HTTPResponse: &http.Response{StatusCode: http.StatusOK}, - JSON200: &astrov1alpha1.Deploy{Id: "deploy-1", ImageRepository: "repo"}, - }, nil).Once() - - deploy, err := createDeployWithDagBundle("org-id", "deployment-id", "desc", "finance", false, nil, mockV1Alpha1Client) - assert.NoError(t, err) - assert.Equal(t, "deploy-1", deploy.Id) - assert.Equal(t, "repo", deploy.ImageRepository) - mockV1Alpha1Client.AssertExpectations(t) - }) - - t.Run("maps a --dags deploy to the DAG type", func(t *testing.T) { - mockV1Alpha1Client := new(astrov1alpha1_mocks.ClientWithResponsesInterface) - mockV1Alpha1Client.On("CreateDeployWithResponse", mock.Anything, mock.Anything, mock.Anything, mock.MatchedBy(func(req astrov1alpha1.CreateDeployRequest) bool { - return req.Type == astrov1alpha1.CreateDeployRequestTypeDAG - })).Return(&astrov1alpha1.CreateDeployResponse{ - HTTPResponse: &http.Response{StatusCode: http.StatusOK}, - JSON200: &astrov1alpha1.Deploy{Id: "deploy-2"}, - }, nil).Once() - - deploy, err := createDeployWithDagBundle("org-id", "deployment-id", "desc", "finance", true, nil, mockV1Alpha1Client) - assert.NoError(t, err) - assert.Equal(t, "deploy-2", deploy.Id) - mockV1Alpha1Client.AssertExpectations(t) - }) - - t.Run("surfaces a transport error", func(t *testing.T) { - mockV1Alpha1Client := new(astrov1alpha1_mocks.ClientWithResponsesInterface) - mockV1Alpha1Client.On("CreateDeployWithResponse", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, errMock).Once() - - _, err := createDeployWithDagBundle("org-id", "deployment-id", "desc", "finance", false, nil, mockV1Alpha1Client) - assert.ErrorIs(t, err, errMock) - mockV1Alpha1Client.AssertExpectations(t) - }) - - t.Run("surfaces an API error response", func(t *testing.T) { - mockV1Alpha1Client := new(astrov1alpha1_mocks.ClientWithResponsesInterface) - mockV1Alpha1Client.On("CreateDeployWithResponse", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&astrov1alpha1.CreateDeployResponse{ - HTTPResponse: &http.Response{StatusCode: http.StatusBadRequest}, - Body: []byte(`{"message":"the targeted DAG bundle no longer exists"}`), - }, nil).Once() - - _, err := createDeployWithDagBundle("org-id", "deployment-id", "desc", "finance", false, nil, mockV1Alpha1Client) - assert.ErrorContains(t, err, "the targeted DAG bundle no longer exists") - mockV1Alpha1Client.AssertExpectations(t) - }) -} - -func TestToV1Alpha1GitRequest(t *testing.T) { - t.Run("returns nil for nil input", func(t *testing.T) { - assert.Nil(t, toV1Alpha1GitRequest(nil)) - }) - - t.Run("maps every field including the provider", func(t *testing.T) { - account := "acme" - branch := "main" - sha := "abc123" - repo := "dags" - git := &astrov1.CreateDeployGitRequest{ - Account: &account, - Branch: &branch, - CommitSha: sha, - Repo: &repo, - Provider: astrov1.CreateDeployGitRequestProviderGITHUB, - } - - got := toV1Alpha1GitRequest(git) - assert.NotNil(t, got) - assert.Equal(t, &account, got.Account) - assert.Equal(t, &branch, got.Branch) - assert.Equal(t, sha, got.CommitSha) - assert.Equal(t, &repo, got.Repo) - assert.Equal(t, astrov1alpha1.CreateDeployGitRequestProvider(astrov1.CreateDeployGitRequestProviderGITHUB), got.Provider) - }) -} diff --git a/internal/platform/astro/deploy/manifest.go b/internal/platform/astro/deploy/manifest.go index bb18f4bc2..19bb6b1d0 100644 --- a/internal/platform/astro/deploy/manifest.go +++ b/internal/platform/astro/deploy/manifest.go @@ -13,7 +13,7 @@ import ( ) // ManifestDagDeployInput is the resolved input for a project's dags-only deploy. -// The deployment is already chosen; this reuses the 1.x path's dags transport (create +// The deployment is already chosen; this uses the dags transport (create // deploy, upload the tarball, finalize) against the project's dags/ directory. type ManifestDagDeployInput struct { // Login is the Astro login the deploy runs under, whose host the @@ -46,7 +46,7 @@ type ManifestDeployGit struct { Uncommitted bool } -// readManifestDeployGit reads the git metadata a manifest deploy records, under the 1.x path's +// readManifestDeployGit reads the git metadata a manifest deploy records, under Astro CLI 1.x's // rules: none when deploy.git_metadata is off, and none when the tree has // uncommitted changes, since HEAD would not describe the files deployed. The // commit message comes back as the description fallback. @@ -71,13 +71,13 @@ func descriptionOrCommitMessage(description, commitMessage string) string { // DeployManifestDags deploys only the dags/ directory of a project to an already // resolved deployment. It reads the runtime version and type from the // server-side deployment — a project ships no image, so a dags-only deploy -// must fit the image already running — then reuses the 1.x path's dags transport. +// must fit the image already running — then uses the dags transport. // -// Unlike the 1.x path's Deploy(), it neither prints nor exits: it returns a result for -// cmd to render, so the path stays cancellable and ready for --output json. It -// reuses createDeploy and deployDags as they are, and finalizes through a -// print-free helper rather than the 1.x path's finalizeDeploy, which prints. A -// --wait's progress is the one thing it writes, and only to in.Progress. +// It neither prints nor exits: it returns a result for cmd to render, so the +// path stays cancellable and ready for --output json. It uses createDeploy and +// uploadDags as they are, and finalizes through the print-free +// finalizeManifestDeploy. A --wait's progress is the one thing it writes, and +// only to in.Progress. // //nolint:gocritic // value input keeps this seam symmetric with DeployManifestImage func DeployManifestDags(in ManifestDagDeployInput, astroV1Client astrov1.APIClient) (ManifestDagDeployResult, error) { diff --git a/internal/platform/astro/deploy/manifest_image.go b/internal/platform/astro/deploy/manifest_image.go index f04b25bfa..670730bbb 100644 --- a/internal/platform/astro/deploy/manifest_image.go +++ b/internal/platform/astro/deploy/manifest_image.go @@ -69,6 +69,8 @@ type ManifestImageDeployResult struct { DagTarballVersion string URL string Git ManifestDeployGit + // DagDeployEnabled is whether the Deployment takes DAG deploys. + DagDeployEnabled bool } // errNoDocker is the plain, actionable message for the no-Docker user @@ -90,8 +92,8 @@ var ( // DeployManifestImage builds (or adopts) a project's image, pushes it to the // deployment's registry, and finalizes; a "both" deploy also uploads the dags/ -// tarball. It reuses the 1.x transport (createDeploy, the registry push in -// airflow.DockerImage.Push, deployDags, finalize) and, like DeployManifestDags, +// tarball. It uses the shared transport (createDeploy, the registry push in +// airflow.DockerImage.Push, uploadDags, finalize) and, like DeployManifestDags, // neither prints nor exits — it returns a result for cmd to render, and writes // only a --wait's progress, to in.Progress. // @@ -201,6 +203,7 @@ func DeployManifestImage(in ManifestImageDeployInput, astroV1Client astrov1.APIC DagTarballVersion: tarballVersion, URL: dashboardURL(c.Domain, dep.Id, dep.WorkspaceId), Git: gitInfo, + DagDeployEnabled: dep.IsDagDeployEnabled, }, nil } @@ -441,7 +444,7 @@ func downgradeError(tag, currentVersion string, raise func(string) string) error return errors.New(msg) } -// checkRuntimeVersion is the 1.x path's ValidRuntimeVersion without the prints: it returns +// checkRuntimeVersion is Astro CLI 1.x's runtime check without the prints: it returns // a descriptive error instead of printing the reason (and leaving the caller to // exit), so the manifest path stays print-free below cmd — the same move // finalizeManifestDeploy makes for finalize. The rules are identical: no downgrade, @@ -480,8 +483,8 @@ func checkAirflow3Floor(currentVersion, tag string) error { // finalizeManifestDeploy marks a manifest deploy final. It carries the dag tarball version // only when one exists (a "both" or dags-only deploy), so an image-only deploy -// finalizes with an empty request. It is the 1.x path's finalizeDeploy without the -// prints, so the manifest path renders in cmd and stays ready for --output json. +// finalizes with an empty request. It prints nothing, so the deploy renders in +// cmd and stays ready for --output json. func finalizeManifestDeploy(organizationID, deploymentID, deployID, dagTarballVersion string, astroV1Client astrov1.APIClient) error { req := astrov1.FinalizeDeployRequest{} if dagTarballVersion != "" { diff --git a/internal/platform/astro/deploy/testfiles/.dockerignore b/internal/platform/astro/deploy/testfiles/.dockerignore deleted file mode 100644 index e69de29bb..000000000 diff --git a/internal/platform/astro/deploy/testfiles/.env b/internal/platform/astro/deploy/testfiles/.env deleted file mode 100644 index e69de29bb..000000000 diff --git a/internal/platform/astro/deploy/testfiles/Dockerfile b/internal/platform/astro/deploy/testfiles/Dockerfile deleted file mode 100644 index aabaf876e..000000000 --- a/internal/platform/astro/deploy/testfiles/Dockerfile +++ /dev/null @@ -1 +0,0 @@ -FROM quay.io/astronomer/astro-runtime:4.2.5 diff --git a/internal/platform/astro/deploy/testfiles/Dockerfile.invalid b/internal/platform/astro/deploy/testfiles/Dockerfile.invalid deleted file mode 100644 index 24899cb5c..000000000 --- a/internal/platform/astro/deploy/testfiles/Dockerfile.invalid +++ /dev/null @@ -1,2 +0,0 @@ -FROM quay.io/astronomer/astro-runtime:3.0.2 -CMD ["echo", 1] diff --git a/pkg/ansi/ansi.go b/pkg/ansi/ansi.go index e89519b98..578ffaa0c 100644 --- a/pkg/ansi/ansi.go +++ b/pkg/ansi/ansi.go @@ -28,11 +28,6 @@ func Color() aurora.Aurora { return aurora.NewAurora(shouldUseColors()) } -// Red returns text colored red -func Red(text string) string { - return color.Sprintf(color.Red(text)) -} - // Green returns text colored green func Green(text string) string { return color.Sprintf(color.Green(text)) diff --git a/pkg/fileutil/files.go b/pkg/fileutil/files.go index bbd89acbd..e13f27bd5 100644 --- a/pkg/fileutil/files.go +++ b/pkg/fileutil/files.go @@ -245,22 +245,13 @@ func ReadFileToString(filename string) (string, error) { return string(data), nil } -// This function finds all files of a specific extension -func GetFilesWithSpecificExtension(folderPath, ext string) []string { - files, _ := filesWithExtension(folderPath, ext, false) //nolint:errcheck // a path it cannot read is skipped - return files -} - -// FilesWithExtension is GetFilesWithSpecificExtension failing on a path -// under folderPath it cannot read, rather than counting it as having none. +// FilesWithExtension finds the names of the files under folderPath whose +// names match ext. It fails on a path under folderPath it cannot read, rather +// than counting it as having none. func FilesWithExtension(folderPath, ext string) ([]string, error) { - return filesWithExtension(folderPath, ext, true) -} - -func filesWithExtension(folderPath, ext string, strict bool) ([]string, error) { var files []string err := filepath.Walk(folderPath, func(path string, f os.FileInfo, walkErr error) error { - if walkErr != nil && strict { + if walkErr != nil { return walkErr } if f != nil && !f.IsDir() { diff --git a/pkg/fileutil/files_test.go b/pkg/fileutil/files_test.go index dbe246a59..d522cfb66 100644 --- a/pkg/fileutil/files_test.go +++ b/pkg/fileutil/files_test.go @@ -356,7 +356,7 @@ func (s *Suite) TestReadFileToString() { } } -func (s *Suite) TestGetFilesWithSpecificExtension() { +func (s *Suite) TestFilesWithExtension() { filePath := "./test.py" content := "testing" WriteStringToFile(filePath, content) @@ -378,7 +378,8 @@ func (s *Suite) TestGetFilesWithSpecificExtension() { } for _, tt := range tests { s.Run(tt.name, func() { - files := GetFilesWithSpecificExtension(tt.args.folderPath, tt.args.ext) + files, err := FilesWithExtension(tt.args.folderPath, tt.args.ext) + s.NoError(err) s.Equal(expectedFiles, files) }) } diff --git a/pkg/util/util.go b/pkg/util/util.go index a7b001044..52d5574cc 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -34,18 +34,6 @@ func Contains(elems []string, v string) bool { return false } -// exists returns whether the given file or directory exists -func Exists(path string) (bool, error) { - _, err := os.Stat(path) - if err == nil { - return true, nil - } - if os.IsNotExist(err) { - return false, nil - } - return false, err -} - // Base64URLEncode delegates to astroauth.Base64URLEncode. // See https://datatracker.ietf.org/doc/html/rfc4648#section-5 func Base64URLEncode(arg []byte) string { diff --git a/pkg/util/util_test.go b/pkg/util/util_test.go index 0752b7a85..a1147e584 100644 --- a/pkg/util/util_test.go +++ b/pkg/util/util_test.go @@ -3,7 +3,6 @@ package util import ( "testing" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) @@ -45,41 +44,6 @@ func (s *Suite) TestContains() { } } -func (s *Suite) TestExists() { - type args struct { - path string - } - tests := []struct { - name string - args args - want bool - errAssertion assert.ErrorAssertionFunc - }{ - { - name: "valid case", - args: args{"./util_test.go"}, - want: true, - errAssertion: assert.NoError, - }, - { - name: "invalid case", - args: args{"./test.go"}, - want: false, - errAssertion: assert.NoError, - }, - } - for _, tt := range tests { - s.Run(tt.name, func() { - got, err := Exists(tt.args.path) - if !tt.errAssertion(s.T(), err) { - return - } - - s.Equal(tt.want, got) - }) - } -} - func (s *Suite) TestBase64URLEncode() { type args struct { arg []byte