test: enable local feature flag evaluation compliance

.github/workflows/sdk-compliance.yml

@@ -12,20 +12,37 @@ on:
       - main
 
 jobs:
+  adapter-tests:
+    name: Compliance adapter protocol tests
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: "3.12"
+      - name: Install adapter and test dependencies
+        run: python -m pip install -e . -r sdk_compliance_adapter/requirements.txt pytest pytest-timeout pytest-asyncio
+      - name: Test adapter protocol
+        run: python -m pytest sdk_compliance_adapter/test_adapter.py --timeout=30
+
   compliance:
     name: PostHog SDK compliance tests (capture v0)
-    uses: PostHog/posthog-sdk-test-harness/.github/workflows/test-sdk-action.yml@03d972e49be84402c491324320b0a0f38c2ddc53
+    uses: PostHog/posthog-sdk-test-harness/.github/workflows/test-sdk-action.yml@1d6b197ef46758f577f470535bb3d70e37a15fa2 # 1.1.1
     with:
       adapter-dockerfile: "sdk_compliance_adapter/Dockerfile"
       adapter-context: "."
-      test-harness-version: "0.10.0"
+      test-harness-version: "1.1.1"
+      continue-on-error: false
       report-name: "sdk-compliance-report-v0"
 
   compliance-v1:
     name: PostHog SDK compliance tests (capture v1)
-    uses: PostHog/posthog-sdk-test-harness/.github/workflows/test-sdk-action.yml@03d972e49be84402c491324320b0a0f38c2ddc53
+    uses: PostHog/posthog-sdk-test-harness/.github/workflows/test-sdk-action.yml@1d6b197ef46758f577f470535bb3d70e37a15fa2 # 1.1.1
     with:
       adapter-dockerfile: "sdk_compliance_adapter/Dockerfile.v1"
       adapter-context: "."
-      test-harness-version: "0.10.0"
+      test-harness-version: "1.1.1"
+      continue-on-error: false
       report-name: "sdk-compliance-report-v1"

sdk_compliance_adapter/CONTRIBUTING.md

@@ -4,7 +4,16 @@ This package contains the PostHog Python SDK compliance adapter used with the Po
 
 ## Running tests
 
-Tests run automatically in CI via GitHub Actions.
+Tests run automatically in CI via GitHub Actions against harness **1.1.1**, for both capture protocols. Both jobs opt into the local-evaluation suite and fail on compliance regressions.
+
+Run adapter protocol tests from the repository root in an activated virtual environment:
+
+```bash
+python -m pip install -e . -r sdk_compliance_adapter/requirements.txt pytest pytest-timeout pytest-asyncio
+python -m pytest sdk_compliance_adapter/test_adapter.py --timeout=30
+```
+
+These tests exercise the real SDK loader/evaluator with controlled transports, including failed/late reloads, local false versus inconclusive, and forced remote evaluation.
 
 ### Locally with Docker Compose
 
@@ -35,7 +44,7 @@ docker run -d --name sdk-adapter --network test-network -p 8080:8080 posthog-pyt
 docker run --rm \
   --name test-harness \
   --network test-network \
-  ghcr.io/posthog/sdk-test-harness:0.10.0 \
+  ghcr.io/posthog/sdk-test-harness:1.1.1 \
   run --adapter-url http://sdk-adapter:8080 --mock-url http://test-harness:8081
 
 # Cleanup

sdk_compliance_adapter/README.md

@@ -24,6 +24,8 @@ The adapter implements the standard SDK adapter interface defined in the [test h
 - `POST /capture` - Capture an event
 - `POST /flush` - Flush pending events
 - `GET /state` - Return internal state
+- `POST /get_feature_flag` - Evaluate a flag locally or remotely
+- `POST /reload_feature_flag_definitions` - Fresh, bounded definitions readiness barrier
 - `POST /reset` - Reset SDK state
 
 ### Key Implementation Details
@@ -34,6 +36,31 @@ The adapter implements the standard SDK adapter interface defined in the [test h
 
 **UUID Tracking**: Extracts and tracks UUIDs from batches to verify deduplication.
 
+### Local feature flag evaluation
+
+Both capture adapters advertise `feature_flags_local_evaluation_v1` for harness
+**1.1.1**. The capability versions the adapter protocol and tests both legacy and
+explicit property matching; it does not change the SDK's default matching mode.
+
+- `/init` maps optional `personal_api_key` to the SDK's `secret_key`. Ordinary
+  capture/remote tests do not need it. Background polling is disabled in the
+  adapter; explicit reloads still use the real SDK definitions loader.
+- `/reload_feature_flag_definitions` takes `timeout_ms` (default 5000, range
+  1–30000). It waits for a fresh successful publication, not merely an existing
+  snapshot. Failed fetches and authorization/quota resets return `ready: false`.
+  A timeout returns HTTP 504; the SDK's in-flight request may finish later on its
+  original Client, and another reload on that Client is rejected while it runs.
+- `/get_feature_flag` with `only_evaluate_locally: true` uses the SDK's local-only
+  result API without emitting flag-called events. A conclusive false has
+  `locally_evaluated: true`; an inconclusive result has `value: null`,
+  `success: false`, and `locally_evaluated: false`, never remote fallback.
+- `force_remote: true` conflicts with local-only mode. When definitions are
+  enabled, forced remote calls use a separate definitions-free SDK Client so
+  they cannot accidentally resolve from local rules. Legacy remote responses
+  and flag-called events are preserved. Reset disposes both Clients.
+
+The adapter is sequential (it does not advertise parallel-test support).
+
 ## Documentation
 
 For complete documentation on the test harness and how to implement adapters, see:

sdk_compliance_adapter/adapter.py

@@ -78,25 +78,26 @@ def __init__(self):
         self.last_error: Optional[str] = None
         self.requests_made: List[RequestInfo] = []
         self.client: Optional[Client] = None
+        self.remote_client: Client | None = None
+        self.reload_thread: threading.Thread | None = None
         self.retry_attempts: Dict[str, int] = {}  # Track retry attempts by batch ID
 
     def reset(self):
         """Reset all state"""
-        client_to_shutdown = None
         with self.lock:
-            client_to_shutdown = self.client
+            clients_to_shutdown = (self.client, self.remote_client)
             self.client = None
-
-        if client_to_shutdown:
-            # Flush and shutdown the existing client outside state.lock.
-            # The patched transport records successful flush requests through
-            # SDKState.record_request(), which also needs state.lock. Holding the
-            # lock while shutdown() waits for the queue to drain can deadlock when
-            # a pending background event is being flushed during test reset.
-            try:
-                client_to_shutdown.shutdown()
-            except Exception as e:
-                logger.warning(f"Error shutting down client: {e}")
+            self.remote_client = None
+            # A timed-out load only owns its old Client, never a replacement.
+            self.reload_thread = None
+
+        for client in clients_to_shutdown:
+            if client:
+                # Flush outside state.lock: transport instrumentation needs it.
+                try:
+                    client.shutdown()
+                except Exception as e:
+                    logger.warning(f"Error shutting down client: {e}")
 
         with self.lock:
             self.pending_events = 0
@@ -309,6 +310,7 @@ def health():
         if is_v1()
         else ["capture_v0", "capture_ai_v0", "encoding_gzip"]
     )
+    capabilities.append("feature_flags_local_evaluation_v1")
     return jsonify(
         {
             "sdk_name": "posthog-python",
@@ -352,21 +354,29 @@ def init():
         # One adapter process speaks one capture protocol, selected by CAPTURE_MODE.
         capture_mode = "v1" if is_v1() else "v0"
 
-        # Create client
-        client = Client(
-            project_api_key=api_key,
-            host=host,
-            flush_at=flush_at,
-            flush_interval=flush_interval,
-            gzip=enable_compression,
-            max_retries=max_retries,
-            debug=False,
-            disable_geoip=disable_geoip,
-            historical_migration=historical_migration,
-            capture_mode=capture_mode,
-        )
-
+        # Explicit reloads exercise the real loader without background polling
+        # racing the harness's per-test definition snapshots.
+        client_options = {
+            "project_api_key": api_key,
+            "host": host,
+            "flush_at": flush_at,
+            "flush_interval": flush_interval,
+            "gzip": enable_compression,
+            "max_retries": max_retries,
+            "debug": False,
+            "disable_geoip": disable_geoip,
+            "historical_migration": historical_migration,
+            "capture_mode": capture_mode,
+            "enable_local_evaluation": False,
+        }
+        personal_api_key = data.get("personal_api_key")
+        client = Client(**client_options, secret_key=personal_api_key)
         state.client = client
+        if personal_api_key:
+            # The SDK has no force-remote switch once definitions are loaded.
+            # A definitions-free Client preserves the real remote API and its
+            # event side effects without mutating the local Client's snapshot.
+            state.remote_client = Client(**client_options)
 
         logger.info(
             f"Initialized SDK with api_key={api_key[:10]}..., host={host}, "
@@ -562,6 +572,70 @@ def get_state():
         return jsonify({"error": str(e)}), 500
 
 
+@app.route("/reload_feature_flag_definitions", methods=["POST"])
+def reload_feature_flag_definitions():
+    """Bound a fresh SDK load and acknowledge only its successful publication."""
+    data = request.json or {}
+    timeout_ms = data.get("timeout_ms", 5000)
+    if type(timeout_ms) is not int or not 1 <= timeout_ms <= 30000:
+        return jsonify(success=False, ready=False, error="Invalid timeout_ms"), 400
+
+    errors = []
+    with state.lock:
+        client = state.client
+        if client is None or not client.personal_api_key:
+            return jsonify(
+                success=False, ready=False, error="A personal_api_key is required"
+            ), 400
+        if state.reload_thread and state.reload_thread.is_alive():
+            return jsonify(
+                success=False,
+                ready=False,
+                error="A definitions reload is still running",
+            ), 409
+        previous_generation = client._flag_definition_published_generation
+
+        def load():
+            try:
+                client.load_feature_flags()
+            except Exception as error:
+                logger.exception("Error reloading feature flag definitions")
+                errors.append(str(error))
+
+        worker = threading.Thread(target=load, daemon=True)
+        state.reload_thread = worker
+        worker.start()
+
+    # The SDK's definitions transport timeout is longer than the adapter's
+    # deadline. Do not block this endpoint on it or start overlapping reloads.
+    worker.join(timeout_ms / 1000)
+    if worker.is_alive():
+        return jsonify(
+            success=False, ready=False, error="Definitions reload timed out"
+        ), 504
+    if errors:
+        return jsonify(success=False, ready=False, error=errors[0]), 502
+    with state.lock:
+        if state.client is not client:
+            return jsonify(
+                success=False, ready=False, error="SDK reset during reload"
+            ), 409
+    # load_feature_flags returns None even on failure. Its publication generation
+    # advances on successful GET/304 and auth/quota resets; the latter clear the
+    # fingerprint. Checking both avoids acknowledging stale or reset definitions.
+    with client._flag_definition_publication_lock:
+        ready = (
+            client._flag_definition_published_generation > previous_generation
+            and bool(client._flag_definition_fingerprint)
+            and client.feature_flags is not None
+        )
+    if not ready:
+        return jsonify(
+            success=False, ready=False, error="Fresh definitions were not loaded"
+        ), 502
+    return jsonify(success=True, ready=True)
+
+
 @app.route("/get_feature_flag", methods=["POST"])
 def get_feature_flag():
     """Evaluate a feature flag"""
@@ -577,14 +651,40 @@ def get_feature_flag():
         groups = data.get("groups")
         group_properties = data.get("group_properties")
         disable_geoip = data.get("disable_geoip")
-        force_remote = data.get("force_remote", True)
+        only_evaluate_locally = data.get("only_evaluate_locally", False)
+        force_remote = data.get("force_remote", not only_evaluate_locally)
 
+        if only_evaluate_locally and force_remote:
+            return jsonify(
+                {"error": "only_evaluate_locally conflicts with force_remote"}
+            ), 400
         if not key:
             return jsonify({"error": "key is required"}), 400
         if not distinct_id:
             return jsonify({"error": "distinct_id is required"}), 400
 
-        value = state.client.get_feature_flag(
+        if only_evaluate_locally:
+            result = state.client.get_feature_flag_result(
+                key,
+                distinct_id,
+                person_properties=person_properties,
+                groups=groups,
+                group_properties=group_properties,
+                disable_geoip=disable_geoip,
+                only_evaluate_locally=True,
+                send_feature_flag_events=False,
+            )
+            # The real local-only SDK API returns None for inconclusive results;
+            # a conclusive false is a FeatureFlagResult, not a cache miss.
+            conclusive = result is not None
+            return jsonify(
+                success=conclusive,
+                value=result.get_value() if result is not None else None,
+                locally_evaluated=conclusive,
+            )
+
+        client = (state.remote_client or state.client) if force_remote else state.client
+        value = client.get_feature_flag(
             key,
             distinct_id,
             person_properties=person_properties,
@@ -598,7 +698,7 @@ def get_feature_flag():
         # the adapter action returns. Otherwise the harness may reset mock-server
         # state for the next test while the background consumer is still flushing,
         # leaking the previous test's event into the next test.
-        state.client.flush()
+        client.flush()
 
         logger.info(f"Feature flag {key} for {distinct_id}: {value}")
 

sdk_compliance_adapter/docker-compose.yml

@@ -23,7 +23,7 @@ services:
 
   # Test harness
   test-harness:
-    image: ghcr.io/posthog/sdk-test-harness:0.10.0
+    image: ghcr.io/posthog/sdk-test-harness:1.1.1
     command: ["run", "--adapter-url", "http://sdk-adapter:8080", "--mock-url", "http://test-harness:8081"]
     networks:
       - test-network