fix: adapt scale-down to Doris 4.1.3

pkg/common/utils/mysql/mysql.go

@@ -186,27 +186,28 @@ func (db *DB) DropBE(nodes []*Backend) error {
 		klog.Infoln("mysql DropBE BE node is empty")
 		return nil
 	}
-	nodesString := fmt.Sprintf(`"%s:%d"`, nodes[0].Host, nodes[0].HeartbeatPort)
-	for _, node := range nodes[1:] {
-		nodesString = nodesString + fmt.Sprintf(`,"%s:%d"`, node.Host, node.HeartbeatPort)
-	}
 
-	alter := fmt.Sprintf("ALTER SYSTEM DROPP BACKEND %s;", nodesString)
-	_, err := db.Exec(alter)
-	return err
+	for _, node := range nodes {
+		alter := fmt.Sprintf(`ALTER SYSTEM DROPP BACKEND "%s:%d";`, node.Host, node.HeartbeatPort)
+		if _, err := db.Exec(alter); err != nil {
+			return err
+		}
+	}
+	return nil
 }
 
 func (db *DB) DropObserver(nodes []*Frontend) error {
 	if len(nodes) == 0 {
 		klog.Infoln("DropObserver observer node is empty")
 		return nil
 	}
-	var alter string
 	for _, node := range nodes {
-		alter = alter + fmt.Sprintf(`ALTER SYSTEM DROP OBSERVER "%s:%d";`, node.Host, node.EditLogPort)
+		alter := fmt.Sprintf(`ALTER SYSTEM DROP OBSERVER "%s:%d";`, node.Host, node.EditLogPort)
+		if _, err := db.Exec(alter); err != nil {
+			return fmt.Errorf("drop observer %s:%d failed: %w", node.Host, node.EditLogPort, err)
+		}
 	}
-	_, err := db.Exec(alter)
-	return err
+	return nil
 }
 
 func (db *DB) GetObservers() ([]*Frontend, error) {

pkg/common/utils/mysql/mysql_test.go

@@ -20,6 +20,9 @@ package mysql
 import (
 	_ "crypto/tls"
 	"database/sql/driver"
+	"errors"
+	"fmt"
+	"regexp"
 	"strconv"
 	"testing"
 
@@ -34,9 +37,9 @@ func Test_ShowFrontends(t *testing.T) {
 	}
 
 	columns := []string{"Name", "Host", "EditLogPort", "HttpPort", "QueryPort", "RpcPort", "ArrowFlightSqlPort", "Role", "IsMaster",
-		"ClusterId", "Join", "Alive", "ReplayedJournalId", "LastStartTime", "LastHeartbeat", "IsHelper", "ErrMsg", "Version", "CurrentConnected"}
+		"ClusterId", "Join", "Alive", "ReplayedJournalId", "LastStartTime", "LastHeartbeat", "IsHelper", "ErrMsg", "Version", "CurrentConnected", "LiveSince", "FutureUnknownColumn"}
 	values := []driver.Value{"fe_36d7bccc_d358_4dfd_ad4c_6e988f94f12d", "doriscluster-sample-fe-0.doriscluster-sample-fe-internal.default.svc.cluster.local", 9010, 8030, 9030, 9020, -1, "FOLLOWER", true, "1807668748", true, true, "15443", "2024-08-21 10:04:29",
-		"2024-08-22 07:29:55", true, "", "doris-2.1.5-rc02-d5a02e095d", "Yes"}
+		"2024-08-22 07:29:55", true, "", "doris-2.1.5-rc02-d5a02e095d", "Yes", "2024-08-21 10:04:29", "ignored"}
 	mock.ExpectQuery("show frontends").WillReturnRows(sqlmock.NewRows(columns).AddRows(values))
 	dorisdb := sqlx.NewDb(mysql_db, "mysql")
 	db := &DB{
@@ -48,18 +51,21 @@ func Test_ShowFrontends(t *testing.T) {
 		t.Errorf("show frontends failed, %s", err.Error())
 	}
 	if len(fts) != 1 {
-		t.Errorf("show frontends failed, not retun one frontend.")
+		t.Fatalf("show frontends failed, expected one frontend, got %d", len(fts))
+	}
+	if fts[0].Host != "doriscluster-sample-fe-0.doriscluster-sample-fe-internal.default.svc.cluster.local" || fts[0].Role != "FOLLOWER" {
+		t.Errorf("show frontends failed, known fields were not mapped: %+v", fts[0])
 	}
 }
 
 func Test_ShowBackends(t *testing.T) {
 	columns := []string{"BackendId", "Host", "HeartbeatPort", "BePort", "HttpPort", "BrpcPort", "ArrowFlightSqlPort", "LastStartTime",
 		"LastHeartbeat", "Alive", "SystemDecommissioned", "TabletNum", "DataUsedCapacity", "TrashUsedCapacity", "AvailCapacity", "TotalCapacity", "UsedPct", "MaxDiskUsedPct",
-		"RemoteUsedCapacity", "Tag", "ErrMsg", "Version", "Status", "HeartbeatFailureCounter", "NodeRole"}
+		"RemoteUsedCapacity", "Tag", "ErrMsg", "Version", "Status", "HeartbeatFailureCounter", "NodeRole", "LiveSince", "FutureUnknownColumn"}
 	values := []driver.Value{"10009", "doriscluster-sample-be-0.doriscluster-sample-be-internal.default.svc.cluster.local", 9050, 9060, 8040, 8060, -1, "2024-08-21 10:05:37",
 		"2024-08-22 08:29:46", true, false, 24, "0.000", "0.000", "74.619 GB", "439.037 GB", "83.00 %", "83.00 %", "0.000",
 		"{\"location\" : \"default\"}", "", "doris-2.1.5-rc02-d5a02e095d", "{\"lastSuccessReportTabletsTime\":\"2024-08-22 08:29:09\",\"lastStreamLoadTime\":-1,\"isQueryDisabled\":false,\"isLoadDisabled\":false}",
-		0, "mix"}
+		0, "mix", "2024-08-21 10:05:37", "ignored"}
 	mysql_db, mock, err := sqlmock.New()
 	if err != nil {
 		t.Errorf("sqlmock new failed %s", err.Error())
@@ -76,7 +82,10 @@ func Test_ShowBackends(t *testing.T) {
 		t.Errorf("show backends failed, %s", err.Error())
 	}
 	if len(bds) != 1 {
-		t.Errorf("show backends failed, not return one backend.")
+		t.Fatalf("show backends failed, expected one backend, got %d", len(bds))
+	}
+	if bds[0].BackendID != "10009" || bds[0].NodeRole != "mix" {
+		t.Errorf("show backends failed, known fields were not mapped: %+v", bds[0])
 	}
 }
 
@@ -143,11 +152,10 @@ func Test_DecommissionBE(t *testing.T) {
 }
 
 func Test_DropObserver(t *testing.T) {
-	version := "doris-2.1.5-rc02-d5a02e095d"
-	startTime := "2024-08-21 10:04:29"
-	heartbeat := "2024-08-22 07:29:55"
-	values := []*Frontend{{"fe_36d7bccc_d358_4dfd_ad4c_6e988f94f12d", "doriscluster-sample-fe-0.doriscluster-sample-fe-internal.default.svc.cluster.local", 9010, 8030, 9030, 9020, -1, "FOLLOWER", true, "1807668748", true, true, "15443", &startTime,
-		&heartbeat, true, "", &version, "Yes"}}
+	values := []*Frontend{
+		{Host: "doriscluster-sample-fe-4.doriscluster-sample-fe-internal.default.svc.cluster.local", EditLogPort: 9010},
+		{Host: "doriscluster-sample-fe-3.doriscluster-sample-fe-internal.default.svc.cluster.local", EditLogPort: 9010},
+	}
 
 	tests := [][]*Frontend{
 		{},
@@ -158,7 +166,10 @@ func Test_DropObserver(t *testing.T) {
 	if err != nil {
 		t.Errorf("sqlmock new failed %s", err.Error())
 	}
-	mock.ExpectExec("ALTER SYSTEM DROP OBSERVER").WillReturnResult(sqlmock.NewResult(1, 1))
+	mock.ExpectExec(regexp.QuoteMeta("ALTER SYSTEM DROP OBSERVER \"doriscluster-sample-fe-4.doriscluster-sample-fe-internal.default.svc.cluster.local:9010\";")).
+		WillReturnResult(sqlmock.NewResult(1, 1))
+	mock.ExpectExec(regexp.QuoteMeta("ALTER SYSTEM DROP OBSERVER \"doriscluster-sample-fe-3.doriscluster-sample-fe-internal.default.svc.cluster.local:9010\";")).
+		WillReturnResult(sqlmock.NewResult(1, 1))
 	dorisdb := sqlx.NewDb(mysql_db, "mysql")
 	db := &DB{
 		DB: dorisdb,
@@ -173,6 +184,9 @@ func Test_DropObserver(t *testing.T) {
 			}
 		})
 	}
+	if err := mock.ExpectationsWereMet(); err != nil {
+		t.Fatalf("drop observer expectations were not met: %s", err)
+	}
 }
 
 func Test_GetObservers(t *testing.T) {
@@ -228,36 +242,49 @@ func Test_GetFollowers(t *testing.T) {
 }
 
 func Test_DropBE(t *testing.T) {
-	tests := [][]*Backend{
-		{
-			{
-				Host:          "test",
-				HeartbeatPort: 9050,
-			}, {
-				Host:          "test1",
-				HeartbeatPort: 9050,
-			},
-		},
-		{},
-	}
-
-	mysql_db, mock, err := sqlmock.New()
+	mysqlDB, mock, err := sqlmock.New()
 	if err != nil {
 		t.Errorf("sqlmock new failed %s", err.Error())
 	}
-	mock.ExpectExec("ALTER SYSTEM DROPP BACKEND").WillReturnResult(sqlmock.NewResult(1, 1))
-	dorisdb := sqlx.NewDb(mysql_db, "mysql")
 	db := &DB{
-		DB: dorisdb,
+		DB: sqlx.NewDb(mysqlDB, "mysql"),
 	}
 	defer db.Close()
 
-	for i, test := range tests {
-		t.Run("test"+strconv.Itoa(i), func(t *testing.T) {
-			err = db.DropBE(test)
-			if err != nil {
-				t.Errorf("test decommission failed, err=%s", err.Error())
-			}
-		})
+	nodes := []*Backend{
+		{Host: "test", HeartbeatPort: 9050},
+		{Host: "test1", HeartbeatPort: 9050},
+	}
+	for _, node := range nodes {
+		query := regexp.QuoteMeta(fmt.Sprintf(`ALTER SYSTEM DROPP BACKEND "%s:%d";`, node.Host, node.HeartbeatPort))
+		mock.ExpectExec(query).WillReturnResult(sqlmock.NewResult(1, 1))
+	}
+	if err := db.DropBE(nodes); err != nil {
+		t.Fatalf("drop backends failed: %v", err)
+	}
+	if err := db.DropBE(nil); err != nil {
+		t.Fatalf("drop empty backends failed: %v", err)
+	}
+	if err := mock.ExpectationsWereMet(); err != nil {
+		t.Fatalf("unmet sql expectations: %v", err)
+	}
+}
+
+func TestDropBEReturnsUnexpectedError(t *testing.T) {
+	mysqlDB, mock, err := sqlmock.New()
+	if err != nil {
+		t.Fatalf("sqlmock new failed: %v", err)
+	}
+	db := &DB{DB: sqlx.NewDb(mysqlDB, "mysql")}
+	defer db.Close()
+
+	query := regexp.QuoteMeta(`ALTER SYSTEM DROPP BACKEND "test:9050";`)
+	mock.ExpectExec(query).WillReturnError(errors.New("access denied"))
+
+	if err := db.DropBE([]*Backend{{Host: "test", HeartbeatPort: 9050}}); err == nil {
+		t.Fatal("expected unexpected DropBE error to be returned")
+	}
+	if err := mock.ExpectationsWereMet(); err != nil {
+		t.Fatalf("unmet sql expectations: %v", err)
 	}
 }

pkg/controller/sub_controller/disaggregated_cluster/computegroups/controller.go

@@ -251,43 +251,30 @@ func (dcgs *DisaggregatedComputeGroupsController) reconcileStatefulset(ctx conte
 		return &sc.Event{Type: sc.EventWarning, Reason: sc.CGStorageTemplateImmutable, Message: msg}, errors.New(msg)
 	}
 
-	err := dcgs.preApplyStatefulSet(ctx, st, &est, cluster, cg)
-	if err != nil {
-		klog.Errorf("disaggregatedComputeGroupsController reconcileStatefulset preApplyStatefulSet namespace=%s name=%s failed, err=%s", st.Namespace, st.Name, err.Error())
-		return &sc.Event{Type: sc.EventWarning, Reason: sc.CGSqlExecFailed, Message: err.Error()}, err
-	}
-
-	// be decimmission processing, skip apply statefulset.
-	if skipApplyStatefulset(cluster, cg) {
-		return nil, nil
+	// Direct-drop scale-down is owned by the graceful state machine when the image
+	// supports it. Decommissioning keeps the legacy pre-apply ordering.
+	gracefulInProgress := false
+	if !cluster.Spec.EnableDecommission {
+		gracefulInProgress = dcgs.reconcileGracefulStatefulSet(ctx, st, &est, cluster, cg)
 	}
 
-	// Graceful two-phase restart/shutdown: check if we need to perform a graceful action.
-	// This must happen after preApply but before the actual StatefulSet apply.
-	if dcgs.RestConfig != nil {
-		var cgStatus *dv1.ComputeGroupStatus
-		for i := range cluster.Status.ComputeGroupStatuses {
-			if cluster.Status.ComputeGroupStatuses[i].UniqueId == cg.UniqueId {
-				cgStatus = &cluster.Status.ComputeGroupStatuses[i]
-				break
-			}
+	if !gracefulInProgress {
+		err := dcgs.preApplyStatefulSet(ctx, st, &est, cluster, cg)
+		if err != nil {
+			klog.Errorf("disaggregatedComputeGroupsController reconcileStatefulset preApplyStatefulSet namespace=%s name=%s failed, err=%s", st.Namespace, st.Name, err.Error())
+			return &sc.Event{Type: sc.EventWarning, Reason: sc.CGSqlExecFailed, Message: err.Error()}, err
 		}
-		if cgStatus != nil {
-			// If a graceful action is already in progress or needs to start,
-			// ensure OnDelete strategy to prevent K8s from auto-deleting pods.
-			skipApply, gracefulErr := dcgs.gracefulRolloutReconcile(ctx, dcgs.RestConfig, st, &est, cluster, cg, cgStatus)
-			if gracefulErr != nil {
-				klog.Errorf("reconcileStatefulset gracefulRolloutReconcile failed: %v", gracefulErr)
-				// Continue with normal reconcile on error, don't block.
-			}
-			if skipApply {
-				// Graceful action is in progress. Apply StatefulSet with OnDelete strategy
-				// so K8s won't auto-delete pods, but still update the template.
-				ensureOnDeleteStrategy(st)
-			}
+
+		// be decommission processing, skip apply statefulset.
+		if skipApplyStatefulset(cluster, cg) {
+			return nil, nil
 		}
 	}
 
+	if cluster.Spec.EnableDecommission {
+		dcgs.reconcileGracefulStatefulSet(ctx, st, &est, cluster, cg)
+	}
+
 	if st.Spec.UpdateStrategy.Type == appv1.OnDeleteStatefulSetStrategyType {
 		dcgs.clearStatefulSetRollingUpdate(ctx, st.Namespace, st.Name)
 	}
@@ -318,6 +305,38 @@ func (dcgs *DisaggregatedComputeGroupsController) reconcileStatefulset(ctx conte
 	return nil, nil
 }
 
+func (dcgs *DisaggregatedComputeGroupsController) reconcileGracefulStatefulSet(
+	ctx context.Context,
+	st, est *appv1.StatefulSet,
+	cluster *dv1.DorisDisaggregatedCluster,
+	cg *dv1.ComputeGroup,
+) bool {
+	if dcgs.RestConfig == nil {
+		return false
+	}
+	var cgStatus *dv1.ComputeGroupStatus
+	for i := range cluster.Status.ComputeGroupStatuses {
+		if cluster.Status.ComputeGroupStatuses[i].UniqueId == cg.UniqueId {
+			cgStatus = &cluster.Status.ComputeGroupStatuses[i]
+			break
+		}
+	}
+	if cgStatus == nil {
+		return false
+	}
+
+	skipApply, err := dcgs.gracefulRolloutReconcile(ctx, dcgs.RestConfig, st, est, cluster, cg, cgStatus)
+	if err != nil {
+		klog.Errorf("reconcileStatefulset gracefulRolloutReconcile failed: %v", err)
+	}
+	if skipApply {
+		// Do not fall through to the legacy DropBE workflow while the graceful
+		// state machine owns this StatefulSet.
+		ensureOnDeleteStrategy(st)
+	}
+	return skipApply
+}
+
 func volumeClaimTemplatesEqual(new, old []corev1.PersistentVolumeClaim) bool {
 	if len(new) != len(old) {
 		return false