cluster: don't wait for storage pods in a maintenance

api/operator/v1/cluster_types_test.go

@@ -4,6 +4,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/utils/ptr"
 
 	vmv1beta1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1"
@@ -79,6 +80,42 @@ func TestVLCluster_AvailableStorageNodeIDs(t *testing.T) {
 	}, vmv1beta1.ClusterComponentSelect, []int32{0, 1, 2})
 }
 
+func TestVTCluster_StorageNodeIDsUnderMaintenance(t *testing.T) {
+	f := func(cr *VTCluster, want sets.Set[int32]) {
+		t.Helper()
+		assert.Equal(t, want, cr.StorageNodeIDsUnderMaintenance())
+	}
+
+	f(&VTCluster{
+		Spec: VTClusterSpec{
+			Storage: &VTStorage{
+				MaintenanceSelectNodeIDs: []int32{1, 3},
+				MaintenanceInsertNodeIDs: []int32{0, 3},
+			},
+		},
+	}, sets.New[int32](3))
+
+	f(&VTCluster{}, nil)
+}
+
+func TestVLCluster_StorageNodeIDsUnderMaintenance(t *testing.T) {
+	f := func(cr *VLCluster, want sets.Set[int32]) {
+		t.Helper()
+		assert.Equal(t, want, cr.StorageNodeIDsUnderMaintenance())
+	}
+
+	f(&VLCluster{
+		Spec: VLClusterSpec{
+			VLStorage: &VLStorage{
+				MaintenanceSelectNodeIDs: []int32{1, 3},
+				MaintenanceInsertNodeIDs: []int32{0, 3},
+			},
+		},
+	}, sets.New[int32](3))
+
+	f(&VLCluster{}, nil)
+}
+
 func TestVLSingle_PrefixedName(t *testing.T) {
 	f := func(name string, omit bool, want string) {
 		t.Helper()

api/operator/v1/vlcluster_types.go

@@ -649,9 +649,21 @@ type VLStorage struct {
 	// MaintenanceInsertNodeIDs - excludes given node ids from insert requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
 	// lets say, you have pod-0, pod-1, pod-2, pod-3. to exclude pod-0 and pod-3 from insert routing, define nodeIDs: [0,3].
 	// Useful at storage expanding, when you want to rebalance some data at cluster.
+	//
+	// A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic.
+	// With the default OnDelete rollingUpdateStrategy, the operator doesn't wait for its pod to become ready
+	// and defers changes to the pod and its volumes until the id is removed from either list.
+	// With RollingUpdate, Kubernetes OrderedReady pod management can still block reconciliation while the pod is not ready.
+	// See https://docs.victoriametrics.com/operator/resources/vlcluster/#pods-excluded-from-all-routing
 	// +optional
 	MaintenanceInsertNodeIDs []int32 `json:"maintenanceInsertNodeIDs,omitempty"`
 	// MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
+	//
+	// A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic.
+	// With the default OnDelete rollingUpdateStrategy, the operator doesn't wait for its pod to become ready
+	// and defers changes to the pod and its volumes until the id is removed from either list.
+	// With RollingUpdate, Kubernetes OrderedReady pod management can still block reconciliation while the pod is not ready.
+	// See https://docs.victoriametrics.com/operator/resources/vlcluster/#pods-excluded-from-all-routing
 	// +optional
 	MaintenanceSelectNodeIDs []int32 `json:"maintenanceSelectNodeIDs,omitempty"`
 
@@ -902,6 +914,17 @@ func (cr *VLCluster) AvailableStorageNodeIDs(kind vmv1beta1.ClusterComponent) []
 	return result
 }
 
+// StorageNodeIDsUnderMaintenance returns the vlstorage pod ordinals excluded from both
+// insert and select routing via MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs.
+func (cr *VLCluster) StorageNodeIDsUnderMaintenance() sets.Set[int32] {
+	if cr.Spec.VLStorage == nil {
+		return nil
+	}
+	insertNodes := sets.New(cr.Spec.VLStorage.MaintenanceInsertNodeIDs...)
+	selectNodes := sets.New(cr.Spec.VLStorage.MaintenanceSelectNodeIDs...)
+	return insertNodes.Intersection(selectNodes)
+}
+
 // LastSpecUpdated compares spec with last applied spec stored, replaces old spec and returns true if it's updated
 func (cr *VLCluster) LastSpecUpdated() bool {
 	updated := cr.Status.LastAppliedSpec == nil || !equality.Semantic.DeepEqual(&cr.Spec, cr.Status.LastAppliedSpec)

api/operator/v1/vtcluster_types.go

@@ -512,9 +512,21 @@ type VTStorage struct {
 	// MaintenanceInsertNodeIDs - excludes given node ids from insert requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
 	// lets say, you have pod-0, pod-1, pod-2, pod-3. to exclude pod-0 and pod-3 from insert routing, define nodeIDs: [0,3].
 	// Useful at storage expanding, when you want to rebalance some data at cluster.
+	//
+	// A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic.
+	// With the default OnDelete rollingUpdateStrategy, the operator doesn't wait for its pod to become ready
+	// and defers changes to the pod and its volumes until the id is removed from either list.
+	// With RollingUpdate, Kubernetes OrderedReady pod management can still block reconciliation while the pod is not ready.
+	// See https://docs.victoriametrics.com/operator/resources/vtcluster/#pods-excluded-from-all-routing
 	// +optional
 	MaintenanceInsertNodeIDs []int32 `json:"maintenanceInsertNodeIDs,omitempty"`
 	// MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
+	//
+	// A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic.
+	// With the default OnDelete rollingUpdateStrategy, the operator doesn't wait for its pod to become ready
+	// and defers changes to the pod and its volumes until the id is removed from either list.
+	// With RollingUpdate, Kubernetes OrderedReady pod management can still block reconciliation while the pod is not ready.
+	// See https://docs.victoriametrics.com/operator/resources/vtcluster/#pods-excluded-from-all-routing
 	// +optional
 	MaintenanceSelectNodeIDs []int32 `json:"maintenanceSelectNodeIDs,omitempty"`
 
@@ -770,6 +782,17 @@ func (cr *VTCluster) AvailableStorageNodeIDs(kind vmv1beta1.ClusterComponent) []
 	return result
 }
 
+// StorageNodeIDsUnderMaintenance returns the storage pod ordinals excluded from both
+// insert and select routing via MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs.
+func (cr *VTCluster) StorageNodeIDsUnderMaintenance() sets.Set[int32] {
+	if cr.Spec.Storage == nil {
+		return nil
+	}
+	insertNodes := sets.New(cr.Spec.Storage.MaintenanceInsertNodeIDs...)
+	selectNodes := sets.New(cr.Spec.Storage.MaintenanceSelectNodeIDs...)
+	return insertNodes.Intersection(selectNodes)
+}
+
 // LastSpecUpdated compares spec with last applied spec stored, replaces old spec and returns true if it's updated
 func (cr *VTCluster) LastSpecUpdated() bool {
 	updated := cr.Status.LastAppliedSpec == nil || !equality.Semantic.DeepEqual(&cr.Spec, cr.Status.LastAppliedSpec)

api/operator/v1beta1/vmcluster_types.go

@@ -589,9 +589,21 @@ type VMStorage struct {
 	// MaintenanceInsertNodeIDs - excludes given node ids from insert requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
 	// lets say, you have pod-0, pod-1, pod-2, pod-3. to exclude pod-0 and pod-3 from insert routing, define nodeIDs: [0,3].
 	// Useful at storage expanding, when you want to rebalance some data at cluster.
+	//
+	// A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic.
+	// With the default OnDelete rollingUpdateStrategy, the operator doesn't wait for its pod to become ready
+	// and defers changes to the pod and its volumes until the id is removed from either list.
+	// With RollingUpdate, Kubernetes OrderedReady pod management can still block reconciliation while the pod is not ready.
+	// See https://docs.victoriametrics.com/operator/resources/vmcluster/#pods-excluded-from-all-routing
 	// +optional
 	MaintenanceInsertNodeIDs []int32 `json:"maintenanceInsertNodeIDs,omitempty"`
 	// MaintenanceSelectNodeIDs - excludes given node ids from select requests routing, must contain pod suffixes - for pod-0, id will be 0 and etc.
+	//
+	// A node id listed in both MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs receives no traffic.
+	// With the default OnDelete rollingUpdateStrategy, the operator doesn't wait for its pod to become ready
+	// and defers changes to the pod and its volumes until the id is removed from either list.
+	// With RollingUpdate, Kubernetes OrderedReady pod management can still block reconciliation while the pod is not ready.
+	// See https://docs.victoriametrics.com/operator/resources/vmcluster/#pods-excluded-from-all-routing
 	// +optional
 	MaintenanceSelectNodeIDs []int32 `json:"maintenanceSelectNodeIDs,omitempty"`
 
@@ -938,6 +950,25 @@ func (cr *VMCluster) AvailableStorageNodeIDs(kind ClusterComponent) []int32 {
 	return result
 }
 
+// StorageNodeIDsUnderMaintenance returns the vmstorage pod ordinals excluded from both
+// insert and select routing via MaintenanceInsertNodeIDs and MaintenanceSelectNodeIDs.
+func (cr *VMCluster) StorageNodeIDsUnderMaintenance() sets.Set[int32] {
+	if cr.Spec.VMStorage == nil {
+		return nil
+	}
+	// with discovery enabled vminsert or vmselect find all vmstorage pods via DNS SRV records and ignore maintenance node ids,
+	// so the listed pods still receive traffic. The webhook rejects such configuration, but it can be disabled.
+	if cr.Spec.VMInsert != nil && cr.Spec.VMInsert.Discovery.OrDefault(cr.Spec.Discovery).enabled() {
+		return nil
+	}
+	if cr.Spec.VMSelect != nil && cr.Spec.VMSelect.Discovery.OrDefault(cr.Spec.Discovery).enabled() {
+		return nil
+	}
+	insertNodes := sets.New(cr.Spec.VMStorage.MaintenanceInsertNodeIDs...)
+	selectNodes := sets.New(cr.Spec.VMStorage.MaintenanceSelectNodeIDs...)
+	return insertNodes.Intersection(selectNodes)
+}
+
 // FinalLabels adds cluster labels to the base labels and filters by prefix if needed
 func (cr *VMCluster) FinalLabels(kind ClusterComponent) map[string]string {
 	v := AddClusterLabels(cr.SelectorLabels(kind), "vm")

api/operator/v1beta1/vmcluster_types_test.go

@@ -6,6 +6,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/utils/ptr"
 )
 
@@ -124,6 +125,74 @@ func TestVMCluster_AvailableStorageNodeIDs(t *testing.T) {
 	}, ClusterComponentSelect, []int32{0, 1, 2})
 }
 
+func TestVMCluster_StorageNodeIDsUnderMaintenance(t *testing.T) {
+	f := func(cr *VMCluster, want sets.Set[int32]) {
+		t.Helper()
+		assert.Equal(t, want, cr.StorageNodeIDsUnderMaintenance())
+	}
+
+	// node 3 is excluded from both insert and select routing
+	f(&VMCluster{
+		Spec: VMClusterSpec{
+			VMStorage: &VMStorage{
+				MaintenanceSelectNodeIDs: []int32{1, 3},
+				MaintenanceInsertNodeIDs: []int32{0, 3},
+			},
+		},
+	}, sets.New[int32](3))
+
+	// no overlap between the two lists
+	f(&VMCluster{
+		Spec: VMClusterSpec{
+			VMStorage: &VMStorage{
+				MaintenanceSelectNodeIDs: []int32{1},
+				MaintenanceInsertNodeIDs: []int32{0},
+			},
+		},
+	}, sets.New[int32]())
+
+	// global discovery enabled, maintenance node ids don't affect routing
+	f(&VMCluster{
+		Spec: VMClusterSpec{
+			Discovery: &VMClusterDiscovery{Enabled: true},
+			VMInsert:  &VMInsert{},
+			VMSelect:  &VMSelect{},
+			VMStorage: &VMStorage{
+				MaintenanceSelectNodeIDs: []int32{3},
+				MaintenanceInsertNodeIDs: []int32{3},
+			},
+		},
+	}, nil)
+
+	// vmselect discovery enabled only
+	f(&VMCluster{
+		Spec: VMClusterSpec{
+			VMInsert: &VMInsert{},
+			VMSelect: &VMSelect{Discovery: &VMClusterDiscovery{Enabled: true}},
+			VMStorage: &VMStorage{
+				MaintenanceSelectNodeIDs: []int32{3},
+				MaintenanceInsertNodeIDs: []int32{3},
+			},
+		},
+	}, nil)
+
+	// global discovery disabled by vminsert and vmselect overrides
+	f(&VMCluster{
+		Spec: VMClusterSpec{
+			Discovery: &VMClusterDiscovery{Enabled: true},
+			VMInsert:  &VMInsert{Discovery: &VMClusterDiscovery{Enabled: false}},
+			VMSelect:  &VMSelect{Discovery: &VMClusterDiscovery{Enabled: false}},
+			VMStorage: &VMStorage{
+				MaintenanceSelectNodeIDs: []int32{3},
+				MaintenanceInsertNodeIDs: []int32{3},
+			},
+		},
+	}, sets.New[int32](3))
+
+	// no VMStorage
+	f(&VMCluster{}, nil)
+}
+
 func TestVMCluster_Validate(t *testing.T) {
 	f := func(spec VMClusterSpec, wantErr bool) {
 		t.Helper()