feat: enable sharding across DBs

.github/workflows/backend-tests-postgres.yml

@@ -59,3 +59,63 @@ jobs:
 
       - name: Run backend test suite on Postgres
         run: make test
+
+  db-sharding:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+
+    services:
+      postgres:
+        image: postgres:16
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: efficientai_test
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U postgres -d efficientai_test"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
+
+    env:
+      PGHOST: localhost
+      PGPORT: 5432
+      PGUSER: postgres
+      PGPASSWORD: postgres
+      DATABASE_URL: postgresql://postgres:postgres@localhost:5432/efficientai_catalog
+      TEST_DATABASE_URL: postgresql://postgres:postgres@localhost:5432/efficientai_test
+      CATALOG_DATABASE_URL: postgresql://postgres:postgres@localhost:5432/efficientai_catalog
+      SHARD_DATABASE_URL_01: postgresql://postgres:postgres@localhost:5432/efficientai_data_01
+      SHARD_DATABASE_URL_02: postgresql://postgres:postgres@localhost:5432/efficientai_data_02
+      SHARDING_INTEGRATION_TEST: "1"
+      REDIS_URL: redis://localhost:6379/0
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: "pip"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e ".[dev]"
+
+      - name: Create catalog and row-shard databases
+        run: |
+          for db in efficientai_catalog efficientai_data_01 efficientai_data_02; do
+            psql -d postgres -v ON_ERROR_STOP=1 -c "DROP DATABASE IF EXISTS ${db};"
+            psql -d postgres -v ON_ERROR_STOP=1 -c "CREATE DATABASE ${db};"
+          done
+
+      - name: Run db_sharding unit tests
+        run: make test-sharding
+
+      - name: Run sharding Postgres integration tests
+        run: make test-sharding-integration

Makefile

@@ -1,4 +1,4 @@
-.PHONY: help install-dev check-pytest test test-docker-db test-unit test-integration test-phase1 test-file test-k
+.PHONY: help install-dev check-pytest test test-docker-db test-unit test-integration test-phase1 test-file test-k test-sharding test-sharding-integration
 
 PYTHON ?= python
 PYTEST ?= $(PYTHON) -m pytest
@@ -19,7 +19,8 @@ help: ## Show available make targets
 	@echo "  make test-integration  - run integration tests (marker: integration)"
 	@echo "  make test-phase1       - run current Phase 1 suites"
 	@echo "  make test-file FILE=...- run a specific test file/path"
-	@echo "  make test-k K=...      - run tests matching expression"
+	@echo "  make test-sharding       - run call-import db_sharding tests (unit; no integration marker)"
+	@echo "  make test-sharding-integration - run 2-shard Postgres integration tests (CI)"
 
 install-dev: ## Install project and dev dependencies
 	$(PYTHON) -m pip install -e ".[dev]"
@@ -57,3 +58,10 @@ test-file: check-pytest ## Run one test module/file; usage: make test-file FILE=
 test-k: check-pytest ## Run tests by keyword expression; usage: make test-k K=password
 	@if [ -z "$(K)" ]; then echo "K is required. Example: make test-k K=password"; exit 1; fi
 	$(PYTEST) tests -k "$(K)" $(PYTEST_FLAGS) $(PYTEST_ARGS)
+
+test-sharding: check-pytest ## Run db_sharding unit tests (excludes integration marker)
+	$(PYTEST) tests/test_db_sharding -m "not integration" $(PYTEST_FLAGS) $(PYTEST_ARGS)
+
+test-sharding-integration: check-pytest ## Run 2-shard Postgres sharding integration tests
+	@if [ -z "$$SHARDING_INTEGRATION_TEST" ]; then export SHARDING_INTEGRATION_TEST=1; fi
+	$(PYTEST) tests/test_db_sharding/test_sharding_postgres_integration.py -m integration $(PYTEST_FLAGS) $(PYTEST_ARGS)