feat: KB/DB versioning and CocoIndex-based incremental workspace ingest

.github/workflows/python-ci.yml

@@ -32,7 +32,103 @@ jobs:
         run: python -m compileall -q datamind
 
       - name: Run test suite
-        run: python -m pytest -q -rs
+        run: python -m pytest -q -rs datamind/tests plugins/datamind-context/tests
 
       - name: Verify real SQLite demo
         run: python -m datamind.scripts.verify_sqlite_demo
+
+  versioning:
+    name: Optional versioning providers (Python 3.12)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: pip
+      - name: Install optional versioning and ingest dependencies
+        run: python -m pip install -e ".[dev,cocoindex,dolt]"
+      - name: Run local versioning, publication and recovery contracts
+        run: >-
+          python scripts/ci_required_tests.py -q
+          datamind/tests/test_kb_versioning.py
+          datamind/tests/test_kb_ingest.py
+          datamind/tests/test_pr28_regressions.py
+
+  databases:
+    name: MySQL and Dolt integration
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    services:
+      mysql:
+        image: mysql:8.4
+        env:
+          MYSQL_ROOT_PASSWORD: datamind_ci
+        ports:
+          - 3306:3306
+        options: >-
+          --health-cmd="mysqladmin ping -h localhost -pdatamind_ci"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=12
+    env:
+      DATAMIND_TEST_MYSQL_DSN: mysql+pymysql://root:datamind_ci@127.0.0.1:3306/
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: pip
+      - name: Install database dependencies
+        run: python -m pip install -e ".[dev,mysql,dolt]"
+      - name: Install pinned Dolt binary
+        shell: bash
+        run: |
+          curl --fail --location --retry 3 https://github.com/dolthub/dolt/releases/download/v2.5.0/dolt-linux-amd64.tar.gz -o /tmp/dolt.tar.gz
+          echo "639b8042d1ea46ebf6b645a002fc787134dbe1dc02888cc51a51d1d3eb557c93  /tmp/dolt.tar.gz" | sha256sum --check
+          tar -xzf /tmp/dolt.tar.gz -C /tmp
+          sudo install /tmp/dolt-linux-amd64/bin/dolt /usr/local/bin/dolt
+          dolt version
+      - name: Run database integration tests without skips
+        run: >-
+          python scripts/ci_required_tests.py -q
+          datamind/tests/test_db_ingest.py
+          datamind/tests/test_db_versioning.py
+
+  package:
+    name: Installed wheel smoke test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: pip
+      - name: Build wheel
+        run: python -m pip install build && python -m build --wheel
+      - name: Install wheel in isolated environment
+        run: |
+          python -m venv /tmp/datamind-wheel-env
+          /tmp/datamind-wheel-env/bin/python -m pip install dist/*.whl
+      - name: Check imports, resources and default providers outside checkout
+        working-directory: /tmp
+        run: /tmp/datamind-wheel-env/bin/python "$GITHUB_WORKSPACE/scripts/ci_smoke_installed.py"
+
+  gate:
+    name: CI gate
+    if: always()
+    needs: [test, versioning, databases, package]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Require all test jobs to succeed
+        env:
+          RESULTS: ${{ toJSON(needs) }}
+        run: |
+          python - <<'PY'
+          import json, os
+          results = json.loads(os.environ["RESULTS"])
+          failed = {name: result["result"] for name, result in results.items() if result["result"] != "success"}
+          if failed:
+              raise SystemExit(f"Required CI jobs did not succeed: {failed}")
+          print("All required CI jobs passed")
+          PY

datamind/capabilities/db/base.py

@@ -104,6 +104,13 @@ async def execute_readonly(
         row_limit: int = 1000,
         timeout_s: float = 10.0,
     ) -> QueryResult:
+        self._validate_readonly_sql(sql, row_limit)
+        bounded = self._bound_query(sql, row_limit + 1)
+        return await self._execute_bounded_readonly(
+            engine, bounded, row_limit=row_limit, timeout_s=timeout_s,
+        )
+
+    def _validate_readonly_sql(self, sql: str, row_limit: int) -> None:
         if not sql or not sql.strip():
             raise CapabilityError("db", "Empty SQL")
         if row_limit < 1:
@@ -117,8 +124,12 @@ async def execute_readonly(
                 f"destructive SQL rejected (leading verb is write/DDL): {sql[:80]!r}"
             )
 
-        bounded = ensure_row_limit(sql, row_limit + 1)
+    def _bound_query(self, sql: str, row_limit: int) -> str:
+        return ensure_row_limit(sql, row_limit)
 
+    async def _execute_bounded_readonly(
+        self, engine: Engine, bounded: str, *, row_limit: int, timeout_s: float,
+    ) -> QueryResult:
         def _run() -> QueryResult:
             start = time.perf_counter()
             with engine.connect() as conn:
@@ -157,3 +168,20 @@ def _before_query(self, conn: Any) -> None:  # noqa: D401
     def _quote_ident(self, name: str) -> str:
         """Default identifier quoting; dialects override for back-ticks etc."""
         return '"' + name.replace('"', '""') + '"'
+
+    async def count_by_source(
+        self,
+        engine: Engine,
+        table: str,
+        *,
+        source_file: str,
+        source_column: str = "_source_file",
+    ) -> int:
+        q = self._quote_ident
+        sql = f"SELECT COUNT(*) FROM {q(table)} WHERE {q(source_column)} = :sf"
+
+        def _run() -> int:
+            with engine.connect() as conn:
+                return int(conn.execute(text(sql), {"sf": source_file}).scalar() or 0)
+
+        return await asyncio.to_thread(_run)

datamind/capabilities/db/providers/__init__.py

@@ -1,6 +1,7 @@
 """DB dialect providers — importing this package registers everything."""
 from __future__ import annotations
 
+from . import dolt  # noqa: F401
 from . import sqlite  # noqa: F401
 
 # MySQL is registered even without pymysql installed — we only fail when