Pr 19 remotesapi

.github/workflows/integration-tests.yml

@@ -8,7 +8,7 @@ on:
       doltdb_engine_version:
         description: "DoltDB engine version to test against"
         required: false
-        default: "1.83.6"
+        default: "2.4.1"
   issue_comment:
     types: [created]
 
@@ -22,7 +22,7 @@ permissions:
 
 env:
   GO_VERSION_FILE: go.mod
-  DOLTDB_ENGINE_VERSION: ${{ github.event.inputs.doltdb_engine_version || '1.83.6' }}
+  DOLTDB_ENGINE_VERSION: ${{ github.event.inputs.doltdb_engine_version || '2.4.1' }}
   KIND_CLUSTER_NAME: dolt
 
 jobs:
@@ -55,7 +55,7 @@ jobs:
 
       - name: Install kind
         run: |
-          KIND_VERSION="v0.30.0"
+          KIND_VERSION="v0.33.0"
           curl -Lo ./kind "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64"
           chmod +x ./kind
           sudo mv ./kind /usr/local/bin/kind

.github/workflows/promote.yml

@@ -0,0 +1,32 @@
+name: Promote Release
+
+on:
+  workflow_call:
+    inputs:
+      tag_name:
+        description: "Release tag to promote (e.g. v0.1.0)"
+        required: true
+        type: string
+  workflow_dispatch:
+    inputs:
+      tag_name:
+        description: "Release tag to promote (e.g. v0.1.0)"
+        required: true
+        type: string
+
+permissions:
+  contents: write
+
+jobs:
+  promote:
+    name: Promote Pre-release
+    runs-on: ubuntu-latest
+    steps:
+      - name: Promote to release
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh release edit "${{ inputs.tag_name }}" \
+            --repo "${{ github.repository }}" \
+            --prerelease=false \
+            --latest

.github/workflows/publish.yml

@@ -0,0 +1,119 @@
+name: Publish Helm Charts
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+    inputs:
+      version:
+        description: "Release version (MAJOR.MINOR.PATCH)"
+        required: true
+        type: string
+      latest:
+        description: "Tag artifacts as latest"
+        required: false
+        type: boolean
+        default: false
+      promote:
+        description: "Promote the release after publishing"
+        required: false
+        type: boolean
+        default: false
+
+permissions:
+  contents: write
+  id-token: write
+
+env:
+  REGISTRY: public.ecr.aws/dolthub
+  IMAGE_NAME: doltdb-operator-image
+
+jobs:
+  release:
+    name: Build and Publish
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          ref: ${{ github.event.release.tag_name || format('v{0}', inputs.version) }}
+
+      - name: Compute version
+        id: version
+        run: |
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            VERSION="${{ inputs.version }}"
+            IS_LATEST="${{ inputs.latest }}"
+          else
+            TAG="${{ github.event.release.tag_name }}"
+            VERSION="${TAG#v}"
+            IS_LATEST="true"
+          fi
+          echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
+          echo "is_latest=${IS_LATEST}" >> "$GITHUB_OUTPUT"
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
+        with:
+          role-to-assume: ${{ secrets.ROLE_TO_ASSUME }}
+          aws-region: us-east-1
+
+      - name: Login to ECR Public
+        run: |
+          aws ecr-public get-login-password --region us-east-1 | \
+            docker login --username AWS --password-stdin public.ecr.aws
+          aws ecr-public get-login-password --region us-east-1 | \
+            helm registry login --username AWS --password-stdin public.ecr.aws
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
+
+      - name: Build and push container image
+        uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.version.outputs.version }}
+            ${{ steps.version.outputs.is_latest == 'true' && format('{0}/{1}:latest', env.REGISTRY, env.IMAGE_NAME) || '' }}
+
+      - name: Setup Helm
+        uses: azure/setup-helm@b9e51907a09c216f16ebe8536097933489208112 # v4.3.0
+
+      - name: Setup crane
+        uses: imjasonh/setup-crane@31b88efe9de28ae0ffa220711af4b60be9435f6e # v0.4
+
+      - name: Update chart image reference
+        run: |
+          VERSION="${{ steps.version.outputs.version }}"
+          sed -i "s|repository: .*|repository: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}|" charts/doltdb-operator/values.yaml
+          sed -i "s|tag: .*|tag: \"${VERSION}\"|" charts/doltdb-operator/values.yaml
+
+      - name: Package and push Helm charts
+        run: |
+          VERSION="${{ steps.version.outputs.version }}"
+          IS_LATEST="${{ steps.version.outputs.is_latest }}"
+
+          helm package charts/doltdb-operator-crds --version "$VERSION" --app-version "$VERSION"
+          helm package charts/doltdb-operator --version "$VERSION" --app-version "$VERSION"
+
+          helm push "doltdb-operator-crds-${VERSION}.tgz" "oci://${{ env.REGISTRY }}"
+          helm push "doltdb-operator-${VERSION}.tgz" "oci://${{ env.REGISTRY }}"
+
+          if [ "$IS_LATEST" = "true" ]; then
+            crane tag "${{ env.REGISTRY }}/doltdb-operator-crds:${VERSION}" latest
+            crane tag "${{ env.REGISTRY }}/doltdb-operator:${VERSION}" latest
+          fi
+
+  promote:
+    name: Promote Release
+    needs: release
+    if: github.event_name == 'release' || inputs.promote
+    uses: ./.github/workflows/promote.yml
+    with:
+      tag_name: ${{ github.event.release.tag_name || format('v{0}', inputs.version) }}
+    secrets: inherit

.github/workflows/release.yml

@@ -0,0 +1,79 @@
+name: Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: "Release version (MAJOR.MINOR.PATCH)"
+        required: true
+        type: string
+
+permissions:
+  contents: write
+
+jobs:
+  release:
+    name: Create Release
+    runs-on: ubuntu-latest
+    steps:
+      - name: Validate version
+        run: |
+          if [[ ! "${{ inputs.version }}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+            echo "::error::Version must be in MAJOR.MINOR.PATCH format (e.g. 0.2.0)"
+            exit 1
+          fi
+
+      - name: Checkout
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          ref: main
+
+      - name: Verify on main branch
+        run: |
+          if [ "$(git rev-parse HEAD)" != "$(git rev-parse origin/main)" ]; then
+            echo "::error::Releases must be created from the main branch"
+            exit 1
+          fi
+
+      - name: Generate GitHub App token
+        id: app-token
+        uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v1.11.7
+        with:
+          app-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+
+      - name: Create release
+        env:
+          GH_TOKEN: ${{ steps.app-token.outputs.token }}
+          VERSION: ${{ inputs.version }}
+        run: |
+          INSTALL_NOTES="$(cat <<EOF
+          ## Installation
+
+          \`\`\`bash
+          # Install CRDs
+          helm install doltdb-operator-crds oci://public.ecr.aws/dolthub/doltdb-operator-crds --version ${VERSION}
+
+          # Install operator
+          helm install doltdb-operator oci://public.ecr.aws/dolthub/doltdb-operator --version ${VERSION}
+          \`\`\`
+
+          ## Container Image
+
+          \`\`\`bash
+          docker pull public.ecr.aws/dolthub/doltdb-operator-image:${VERSION}
+          \`\`\`
+          EOF
+          )"
+
+          gh release create "v${VERSION}" \
+            --title "v${VERSION}" \
+            --generate-notes \
+            --notes-start-tag "" \
+            --prerelease \
+            --latest=false
+
+          GENERATED_NOTES="$(gh release view "v${VERSION}" --json body -q .body)"
+          gh release edit "v${VERSION}" --notes "${GENERATED_NOTES}
+
+          ${INSTALL_NOTES}"

Makefile

@@ -2,7 +2,7 @@
 IMG ?= controller:latest
 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
 ENVTEST_K8S_VERSION = 1.31.0
-DOLTDB_ENGINE_VERSION ?= 1.83.6
+DOLTDB_ENGINE_VERSION ?= 2.4.1
 export DOLTDB_ENGINE_VERSION
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)

README.md

@@ -28,11 +28,11 @@ This operator takes care of all of that. Define your DoltDB cluster as a Kuberne
 ### Install with Helm
 
 ```sh
-# Add the CRDs chart (install separately for lifecycle management)
-helm install doltdb-operator-crds oci://ghcr.io/electronicarts/doltdb-operator-crds
+# Install CRDs (separately for lifecycle management)
+helm install doltdb-operator-crds oci://public.ecr.aws/dolthub/doltdb-operator-crds
 
 # Install the operator
-helm install doltdb-operator oci://ghcr.io/electronicarts/doltdb-operator
+helm install doltdb-operator oci://public.ecr.aws/dolthub/doltdb-operator
 ```
 
 ### Deploy a DoltDB Cluster
@@ -43,7 +43,7 @@ kind: DoltDB
 metadata:
   name: my-dolt
 spec:
-  engineVersion: "1.57.2"
+  engineVersion: "2.4.1"
   image: dolthub/dolt
   replicas: 2
   storage:
@@ -125,8 +125,8 @@ Key options:
 
 | Parameter | Description | Default |
 |-----------|-------------|---------|
-| `image.repository` | Operator container image | — |
-| `image.tag` | Operator image tag | — |
+| `image.repository` | Operator container image | `public.ecr.aws/dolthub/doltdb-operator-image` |
+| `image.tag` | Operator image tag | Chart appVersion |
 | `replicas` | Number of operator replicas | `1` |
 | `installCrds` | Install CRDs with the operator chart | `false` |
 | `installClusterRole` | Create ClusterRole and ClusterRoleBinding | `true` |

api/v1alpha/doltdb_server_types.go

@@ -2,6 +2,13 @@
 
 package v1alpha
 
+// The user-facing remotes API port must not collide with any other port the server listens on.
+// Defaults are spelled out because an omitted parent object is not defaulted.
+// +kubebuilder:validation:XValidation:rule="!has(self.remotesAPI) || self.remotesAPI.port != (has(self.listener) && has(self.listener.port) ? self.listener.port : 3306)",message="remotesAPI.port must differ from listener.port"
+// +kubebuilder:validation:XValidation:rule="!has(self.remotesAPI) || self.remotesAPI.port != (has(self.cluster) && has(self.cluster.remotesAPI) && has(self.cluster.remotesAPI.port) ? self.cluster.remotesAPI.port : 50051)",message="remotesAPI.port must differ from cluster.remotesAPI.port"
+// +kubebuilder:validation:XValidation:rule="!has(self.remotesAPI) || !has(self.mcpServer) || self.remotesAPI.port != (has(self.mcpServer.port) ? self.mcpServer.port : 7007)",message="remotesAPI.port must differ from mcpServer.port"
+// +kubebuilder:validation:XValidation:rule="!has(self.remotesAPI) || !has(self.metrics) || !has(self.metrics.enabled) || !self.metrics.enabled || self.remotesAPI.port != (has(self.metrics.port) ? self.metrics.port : 11228)",message="remotesAPI.port must differ from metrics.port"
+// +kubebuilder:validation:XValidation:rule="!has(self.remotesAPI) || !has(self.profiler) || !has(self.profiler.enablePProf) || !self.profiler.enablePProf || self.remotesAPI.port != 6060",message="remotesAPI.port must differ from the profiler port 6060"
 type Server struct {
 	// Profiler defines the profiler configuration for the DoltDB server.
 	// +optional
@@ -26,6 +33,20 @@ type Server struct {
 	// endpoint is started alongside the SQL server.
 	// +optional
 	MCPServer *MCPServer `json:"mcpServer,omitempty"`
+	// RemotesAPI enables Dolt's user-facing remotes API, which serves clone, fetch
+	// and push of the server's databases. When set, it is started alongside the SQL
+	// server. It is separate from Cluster.RemotesAPI, which carries replication traffic.
+	//
+	// The port is exposed on the standalone Service, or on the primary Service when
+	// replication is enabled. It is not exposed on the reader Service, because Dolt
+	// does not refuse pushes to a standby through this API.
+	//
+	// Clients authenticate with HTTP Basic credentials of a SQL user. Reads require
+	// the CLONE_ADMIN privilege and pushes require SUPER. Credentials are sent in
+	// plaintext, so the port should only be reachable over an encrypted network
+	// (e.g. a service mesh) and restricted with a NetworkPolicy.
+	// +optional
+	RemotesAPI *ServerRemotesAPI `json:"remotesAPI,omitempty"`
 }
 
 type Profiler struct {
@@ -39,6 +60,11 @@ type Behavior struct {
 	// AutoGCBehavior defines the auto GC behavior for the DoltDB server.
 	// +optional
 	AutoGCBehavior AutoGCBehavior `json:"autoGCBehavior,omitempty"`
+	// EventScheduler sets the server's event scheduler. When empty, Dolt's default
+	// applies, which is ON.
+	// +kubebuilder:validation:Enum=ON;OFF
+	// +optional
+	EventScheduler string `json:"eventScheduler,omitempty"`
 }
 
 type AutoGCBehavior struct {
@@ -98,6 +124,25 @@ type RemotesAPI struct {
 	Port int32 `json:"port,omitempty"`
 }
 
+// ServerRemotesAPI defines Dolt's user-facing remotes API.
+type ServerRemotesAPI struct {
+	// Port defines the port for the remotes API.
+	// +kubebuilder:validation:Minimum=1024
+	// +kubebuilder:validation:Maximum=65535
+	// +kubebuilder:default:=50052
+	// +optional
+	Port int32 `json:"port,omitempty"`
+	// ReadOnly makes the remotes API refuse pushes.
+	// +optional
+	ReadOnly bool `json:"readOnly,omitempty"`
+}
+
+func (r *ServerRemotesAPI) ApplyDefaults() {
+	if r.Port == 0 {
+		r.Port = 50052
+	}
+}
+
 // MCPServer defines the MCP server configuration for the DoltDB server.
 type MCPServer struct {
 	// Port defines the HTTP port for the MCP server.
@@ -140,6 +185,9 @@ func (s *Server) ApplyDefaults() {
 	if s.MCPServer != nil {
 		s.MCPServer.ApplyDefaults()
 	}
+	if s.RemotesAPI != nil {
+		s.RemotesAPI.ApplyDefaults()
+	}
 }
 
 func (m *Metrics) ApplyDefaults() {