Move CI provisioning from v1 workspaces to v2 clusters

.github/workflows/code-check.yml

@@ -8,6 +8,12 @@ on:
   workflow_dispatch:
 
 
+# No `concurrency`/`cancel-in-progress`, deliberately. GitHub force-terminates a
+# cancelled job's remaining steps -- `if: always()` included -- after a 5-minute
+# cancellation timeout, and an S-00 cluster refuses DELETE until it is ACTIVE
+# (~460s). A run cancelled in its first few minutes would die holding a cluster
+# it is not yet allowed to delete, with nothing scheduled to reap it. Letting
+# both runs finish costs clusters; cancelling them costs stranded clusters.
 jobs:
   test-coverage:
     runs-on: ubuntu-latest
@@ -16,6 +22,12 @@ jobs:
       contents: read
       actions: write
 
+    # One ledger for the whole job, so the cleanup step at the end can reap
+    # what any of the pytest steps created. See the matching block in
+    # coverage.yml.
+    env:
+      SINGLESTOREDB_TEST_DEPLOYMENT_LOG: ${{ github.workspace }}/deployments.jsonl
+
     services:
       singlestore:
         image: ghcr.io/singlestore-labs/singlestoredb-dev:latest
@@ -29,14 +41,22 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
         with:
-          fetch-depth: 2
-
+          # Full history, because the change detector below diffs against
+          # origin/main, a ref a shallow clone does not create. With
+          # fetch-depth: 2 every diff died on `fatal: bad revision
+          # 'origin/main'`, which the detector read as "nothing changed", so the
+          # management step never ran on a PR.
+          fetch-depth: 0
+
+      # One version, not a matrix: the supported range is covered by
+      # smoke-test.yml and pre-commit.yml, so this tracks the newest final
+      # release rather than the floor in pyproject.toml.
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v7
         with:
-          python-version: "3.10"
+          python-version: "3.14"
           cache: "pip"
 
       - name: Install dependencies
@@ -78,8 +98,8 @@ jobs:
           COMMIT_MSG=$(git log -1 --format='%s' HEAD)
           if [[ "$COMMIT_MSG" =~ ^Prepare\ for\ v[0-9]+\.[0-9]+\.[0-9]+\ release$ ]]; then
             echo "🚀 Release preparation commit detected: $COMMIT_MSG"
-            echo "changes-detected=true" >> $GITHUB_OUTPUT
-            echo "changed-directories=release" >> $GITHUB_OUTPUT
+            echo "changes-detected=true" >> "$GITHUB_OUTPUT"
+            echo "changed-directories=release" >> "$GITHUB_OUTPUT"
             echo ""
             echo "🎯 RESULT: Full test suite will run for release preparation"
             exit 0
@@ -94,10 +114,15 @@ jobs:
 
           for DIR in $MONITORED_DIRS; do
             if [ -d "$DIR" ]; then
-              CHANGED_FILES=$(git diff --name-only $BASE_COMMIT HEAD -- "$DIR" || true)
+              # No `|| true` here: a git failure means the comparison did not
+              # happen, and swallowing it silently downgrades the run to the
+              # no-management path instead of reporting the breakage.
+              CHANGED_FILES=$(git diff --name-only "$BASE_COMMIT" HEAD -- "$DIR")
               if [ -n "$CHANGED_FILES" ]; then
                 echo "✅ Changes detected in: $DIR"
                 echo "Files changed:"
+                # shellcheck disable=SC2001  # prefixing every line, which
+                # ${var//search/replace} cannot do
                 echo "$CHANGED_FILES" | sed 's/^/  - /'
                 CHANGES_FOUND=true
                 if [ -z "$CHANGED_DIRS" ]; then
@@ -115,13 +140,13 @@ jobs:
 
           # Set outputs
           if [ "$CHANGES_FOUND" = true ]; then
-            echo "changes-detected=true" >> $GITHUB_OUTPUT
-            echo "changed-directories=$CHANGED_DIRS" >> $GITHUB_OUTPUT
+            echo "changes-detected=true" >> "$GITHUB_OUTPUT"
+            echo "changed-directories=$CHANGED_DIRS" >> "$GITHUB_OUTPUT"
             echo ""
             echo "🎯 RESULT: Changes detected in monitored directories"
           else
-            echo "changes-detected=false" >> $GITHUB_OUTPUT
-            echo "changed-directories=" >> $GITHUB_OUTPUT
+            echo "changes-detected=false" >> "$GITHUB_OUTPUT"
+            echo "changed-directories=" >> "$GITHUB_OUTPUT"
             echo ""
             echo "🎯 RESULT: No changes in monitored directories"
           fi
@@ -137,8 +162,12 @@ jobs:
 
       - name: Run MySQL protocol tests (with management API)
         if: steps.check-changes.outputs.changes-detected == 'true'
+        # -m 'not management_v1' keeps the v2 management coverage while dropping
+        # the deprecated v1 suite, which coverage.yml runs nightly instead. The
+        # -m 'not management' steps below need no second term: they already
+        # exclude everything v1 deploys.
         run: |
-          pytest -v --cov=singlestoredb --pyargs singlestoredb.tests
+          pytest -v -m 'not management_v1' --cov=singlestoredb --pyargs singlestoredb.tests
         env:
           COVERAGE_FILE: "coverage-mysql.cov"
           SINGLESTOREDB_URL: "root:root@127.0.0.1:3307"
@@ -171,7 +200,7 @@ jobs:
           SINGLESTOREDB_FUSION_ENABLE_HIDDEN: "1"
 
       - name: Run HTTP protocol tests
-        # -n 0 overrides the -n 3 in pyproject.toml's addopts: the HTTP/Data API
+        # -n 0 overrides the -n 2 in pyproject.toml's addopts: the HTTP/Data API
         # run must be serial. Setup goes over SINGLESTOREDB_INIT_DB_URL (MySQL),
         # so load_sql takes its `SET GLOBAL HTTP_PROXY_PORT` + `RESTART PROXY`
         # branch (singlestoredb/tests/utils.py:227) once per worker, and a proxy
@@ -195,3 +224,27 @@ jobs:
           coverage report
           coverage xml
           coverage html
+
+      # if: always() is the whole point -- this has to run when the job fails or
+      # is cancelled, which is what left three clusters billing in run
+      # 35631802648 (see the matching step in coverage.yml). On a PR the
+      # management step above only runs when the change detector fires, so most
+      # runs reach this with an empty ledger and report nothing.
+      #
+      # Best effort, not a guarantee: a cancelled job's remaining steps are
+      # force-terminated after GitHub's 5-minute cancellation timeout, so this
+      # covers a cancel whose clusters are already ACTIVE but not one in the
+      # first few minutes, where DELETE is still refused. That remainder needs
+      # `cleanup_deployments.py --older-than` run by hand.
+      #
+      # The secret sweep alongside it is the same rolling janitor coverage.yml
+      # runs; see the comment on that step.
+      - name: Clean up what the tests left behind
+        if: always()
+        run: |
+          python -m singlestoredb.tests.cleanup_deployments --secrets --yes \
+            || true
+          python -m singlestoredb.tests.cleanup_deployments \
+            --ledger "$SINGLESTOREDB_TEST_DEPLOYMENT_LOG" --yes
+        env:
+          SINGLESTOREDB_MANAGEMENT_TOKEN: ${{ secrets.CLUSTER_API_KEY }}

.github/workflows/coverage.yml

@@ -10,6 +10,12 @@ jobs:
     runs-on: ubuntu-latest
     environment: Base
 
+    # One ledger for the whole job, so the cleanup step below can reap what any
+    # of the pytest steps created. Per job, not shared: a job's sweep can then
+    # only reach records it wrote itself.
+    env:
+      SINGLESTOREDB_TEST_DEPLOYMENT_LOG: ${{ github.workspace }}/deployments.jsonl
+
     services:
       singlestore:
         image: ghcr.io/singlestore-labs/singlestoredb-dev:latest
@@ -22,12 +28,15 @@ jobs:
           ROOT_PASSWORD: "root"
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v7
 
+      # One version, not a matrix: the supported range is covered by
+      # smoke-test.yml and pre-commit.yml, so this tracks the newest final
+      # release rather than the floor in pyproject.toml.
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v7
         with:
-          python-version: "3.10"
+          python-version: "3.14"
           cache: "pip"
 
       - name: Install dependencies
@@ -36,8 +45,11 @@ jobs:
           pip install -e ".[dev]"
 
       - name: Run MySQL protocol tests
+        # -m 'not management_v1' keeps the v2 management coverage while dropping
+        # the deprecated v1 suite; the management-v1-tests job below is where
+        # that runs.
         run: |
-          pytest -v --cov=singlestoredb --pyargs singlestoredb.tests
+          pytest -v -m 'not management_v1' --cov=singlestoredb --pyargs singlestoredb.tests
         env:
           COVERAGE_FILE: "coverage-mysql.cov"
           SINGLESTOREDB_URL: "root:root@127.0.0.1:3307"
@@ -58,7 +70,7 @@ jobs:
           SINGLESTOREDB_FUSION_ENABLE_HIDDEN: "1"
 
       - name: Run HTTP protocol tests
-        # -n 0 overrides the -n 3 in pyproject.toml's addopts: the HTTP/Data API
+        # -n 0 overrides the -n 2 in pyproject.toml's addopts: the HTTP/Data API
         # run must be serial. Setup goes over SINGLESTOREDB_INIT_DB_URL (MySQL),
         # so load_sql takes its `SET GLOBAL HTTP_PROXY_PORT` + `RESTART PROXY`
         # branch (singlestoredb/tests/utils.py:227) once per worker, and a proxy
@@ -82,3 +94,117 @@ jobs:
           coverage report
           coverage xml
           coverage html
+
+      # if: always() is the whole point -- this has to run when the job is
+      # cancelled, the case that produced the leak. Run 35631802648 was
+      # cancelled 19 minutes into TestClusterFusion.setUpClass's
+      # create_cluster(wait_on_active=True, wait_timeout=1200): the log ends at
+      # '##[error]The operation was canceled.' with no pytest summary and no
+      # sweep output, leaving three clusters billing with nothing in the process
+      # having recorded them. The ledger is that record.
+      #
+      # Last step in the job, so it covers every pytest step above it.
+      #
+      # Best effort, not a guarantee: a cancelled job's remaining steps are
+      # force-terminated after GitHub's 5-minute cancellation timeout, and an
+      # S-00 cluster refuses DELETE until it is ACTIVE (~460s). The leak above is
+      # covered, being 19 minutes in; a cancel in the first few minutes would be
+      # killed here still getting 400/409, and needs `cleanup_deployments.py
+      # --older-than` run by hand.
+      - name: Clean up what the tests left behind
+        if: always()
+        run: |
+          # Secrets first, and its status discarded. TestSecrets creates an
+          # org-scoped secret that only its own test body deletes, so a killed
+          # run strands one for good; --secrets is age-guarded, which keeps it
+          # off this run's and off a concurrent job's, so what it removes is
+          # what *earlier* runs stranded. A secret bills nothing, and the
+          # ledger sweep is what this step's status should report.
+          python -m singlestoredb.tests.cleanup_deployments --secrets --yes \
+            || true
+          python -m singlestoredb.tests.cleanup_deployments \
+            --ledger "$SINGLESTOREDB_TEST_DEPLOYMENT_LOG" --yes
+        env:
+          SINGLESTOREDB_MANAGEMENT_TOKEN: ${{ secrets.CLUSTER_API_KEY }}
+
+  # The deprecated v1 management API. management.version defaults to v2, so this
+  # is a legacy gate: it runs here nightly rather than on every PR, and it is
+  # what gets deleted along with management/v1/. Selects both the mocked v1
+  # units and the live v1 deployments, plus test_fusion's v1 WORKSPACE grammar.
+  management-v1-tests:
+    runs-on: ubuntu-latest
+    environment: Base
+
+    # Waits for test-coverage rather than running alongside it, so the v1
+    # workspace groups are never in flight at the same time as the v2 suite's
+    # cluster pool. This is a nightly cron, so the extra wall clock is free.
+    #
+    # Runs even when test-coverage fails: a v2 failure above says nothing about
+    # the v1 endpoints, and skipping v1 for it would hide a v1 regression behind
+    # an unrelated one.
+    #
+    # !cancelled() rather than always(), which stays true through cancellation
+    # too. A cancelled run must not start provisioning workspace groups here:
+    # remaining steps are force-terminated 5 minutes into a cancel, so the sweep
+    # below would be killed while the new deployments were still pre-ACTIVE and
+    # refusing DELETE -- stranding exactly what it exists to clean up.
+    needs: test-coverage
+    if: ${{ !cancelled() }}
+
+    # A ledger of its own: separate runner, separate workspace, and neither
+    # job's sweep can reach the other's records.
+    env:
+      SINGLESTOREDB_TEST_DEPLOYMENT_LOG: ${{ github.workspace }}/deployments.jsonl
+
+    services:
+      singlestore:
+        image: ghcr.io/singlestore-labs/singlestoredb-dev:latest
+        ports:
+          - 3307:3306
+          - 8081:8080
+          - 9081:9081
+        env:
+          SINGLESTORE_LICENSE: ${{ secrets.SINGLESTORE_LICENSE }}
+          ROOT_PASSWORD: "root"
+
+    steps:
+      - uses: actions/checkout@v7
+
+      # As in test-coverage above: newest final release, not the floor.
+      - name: Set up Python
+        uses: actions/setup-python@v7
+        with:
+          python-version: "3.14"
+          cache: "pip"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+
+      - name: Run v1 management API tests
+        # -n 0 overrides the -n 2 in pyproject.toml's addopts, which is tuned for
+        # the v2 suite's shared cluster pool. The v1 classes deploy workspace
+        # groups of their own, so two workers would put twice that in flight.
+        # Serial keeps this job to one deployment at a time.
+        run: |
+          pytest -v -n 0 -m 'management_v1' --pyargs singlestoredb.tests
+        env:
+          SINGLESTOREDB_URL: "root:root@127.0.0.1:3307"
+          SINGLESTOREDB_PURE_PYTHON: 0
+          SINGLESTORE_LICENSE: ${{ secrets.SINGLESTORE_LICENSE }}
+          SINGLESTOREDB_MANAGEMENT_TOKEN: ${{ secrets.CLUSTER_API_KEY }}
+          SINGLESTOREDB_FUSION_ENABLE_HIDDEN: "1"
+
+      # See the matching step in test-coverage for why this is if: always(),
+      # and for what the secret sweep is doing here. The v1 suite deploys
+      # workspace groups, which are the kind that force exists for.
+      - name: Clean up what the tests left behind
+        if: always()
+        run: |
+          python -m singlestoredb.tests.cleanup_deployments --secrets --yes \
+            || true
+          python -m singlestoredb.tests.cleanup_deployments \
+            --ledger "$SINGLESTOREDB_TEST_DEPLOYMENT_LOG" --yes
+        env:
+          SINGLESTOREDB_MANAGEMENT_TOKEN: ${{ secrets.CLUSTER_API_KEY }}

.github/workflows/fusion-docs.yml

@@ -14,12 +14,14 @@ jobs:
       actions: write
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v7
 
-      - name: Set up Python 3.11
-        uses: actions/setup-python@v5
+      # Only drives resources/gen_fusion_handlers_doc.py, so this tracks the
+      # newest final release rather than anything the package supports.
+      - name: Set up Python 3.14
+        uses: actions/setup-python@v7
         with:
-          python-version: 3.11
+          python-version: "3.14"
           cache: "pip"
 
       - name: Install dependencies

.github/workflows/pre-commit.yml

@@ -14,12 +14,13 @@ jobs:
           - "3.11"
           - "3.12"
           - "3.13"
+          - "3.14"
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v7
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v7
       with:
         python-version: ${{ matrix.python-version }}