External project save/load: canonical files, durable execution and independent forks

.github/workflows/tests.yml

@@ -26,6 +26,15 @@ jobs:
       TULDOK_TRAINING_PYTHON: ${{ github.workspace }}/build/workspace-training/bin/python
       TULDOK_NATIVE_READER_PYTHON: ${{ github.workspace }}/build/workspace-training/bin/python
       TULDOK_DATABASE_URL: postgresql://tuldok_test:isolated-ci-fixture@127.0.0.1:5432/tuldok_test
+      TULDOK_PROJECT_FILES_COLD_DATABASE_URL: postgresql://tuldok_project_cold:isolated-cold-fixture@127.0.0.1:5432/tuldok_project_cold
+      TULDOK_QA_OUTPUT_ROOT: build/qa/external-project
+      TULDOK_M2_FREEZE_MANIFEST: build/qa/external-project/product-manifest.json
+      TULDOK_M3_FREEZE_MANIFEST: build/qa/external-project/product-manifest.json
+      TULDOK_M4_FREEZE_MANIFEST: build/qa/external-project/product-manifest.json
+      TULDOK_OWNER_MCP_MANIFEST: build/qa/external-project/product-manifest.json
+      TULDOK_M5_CLIENTS_MANIFEST: build/qa/external-project/product-manifest.json
+      TULDOK_M5_UI_FREEZE_MANIFEST: build/qa/external-project/product-manifest.json
+      TULDOK_NATIVE_PUBLIC_FREEZE_MANIFEST: build/qa/external-project/product-manifest.json
     steps:
       - uses: actions/checkout@v4
       - name: Fetch the exact authored fixture used by canonical Release qualification
@@ -47,6 +56,33 @@ jobs:
       - name: Install independent WAV consumer
         run: sudo apt-get update && sudo apt-get install -y ffmpeg
       - run: python -m pip install -r requirements.txt -r requirements-spatial-qualification.txt
+      - name: Prepare isolated external-project cold reconstruction and exact source subject
+        run: |
+          python - <<'PY'
+          import hashlib, json, os, pathlib, subprocess
+          import psycopg
+          from psycopg import sql
+          source = os.environ['TULDOK_DATABASE_URL']
+          database = psycopg.conninfo.conninfo_to_dict(source)['dbname']
+          # This job owns the disposable service and these fixed fixture names.
+          with psycopg.connect(source, autocommit=True) as connection:
+              connection.execute("CREATE ROLE tuldok_project_cold LOGIN PASSWORD 'isolated-cold-fixture' NOSUPERUSER NOCREATEDB NOCREATEROLE")
+              connection.execute('CREATE DATABASE tuldok_project_cold OWNER tuldok_project_cold')
+              connection.execute(sql.SQL('REVOKE CONNECT ON DATABASE {} FROM PUBLIC').format(sql.Identifier(database)))
+          with psycopg.connect(os.environ['TULDOK_PROJECT_FILES_COLD_DATABASE_URL']) as connection:
+              current, allowed = connection.execute('SELECT current_database(),has_database_privilege(current_user,%s,\'CONNECT\')', (database,)).fetchone()
+              assert current != database and allowed is False
+          paths = subprocess.check_output(['git', 'ls-files', '-z']).decode().split('\0')
+          paths = sorted(p for p in paths if p and ((p.endswith('.py') and '/' not in p) or p.startswith(('static/', 'tests/', 'extension_schemas/', 'tuldok_extension_sdk/'))))
+          subject = {'baseline': subprocess.check_output(['git', 'rev-parse', 'HEAD']).decode().strip(),
+                     'qualification_copy': str(pathlib.Path.cwd()), 'parent_manifest_sha256': None,
+                     'files': {p: hashlib.sha256(pathlib.Path(p).read_bytes()).hexdigest() for p in paths}}
+          target = pathlib.Path(os.environ['TULDOK_M2_FREEZE_MANIFEST'])
+          target.parent.mkdir(parents=True, exist_ok=True)
+          target.write_text(json.dumps(subject, sort_keys=True, separators=(',', ':')) + '\n')
+          print('External-project product files:', len(subject['files']))
+          assert subject['files']
+          PY
       - name: Verify field output preflight and unchanged admission state
         run: python -m unittest discover -s tests -p 'test_run_preflight*.py' -v
       - name: Verify same-attempt publication retention and recovery boundaries
@@ -75,6 +111,27 @@ jobs:
       - run: build/workspace-training/bin/python -m unittest discover -s tests -p test_volume_assets.py
       - run: build/workspace-training/bin/python -m unittest discover -s tests -p test_rheon_cpu_training.py
       - run: build/workspace-training/bin/python -m unittest discover -s tests -p test_rheon_evaluation.py
+      - name: Verify actual native project Run fork Study custody and cold accounting
+        env:
+          PYTHONPATH: tests
+          TULDOK_NATIVE_PROJECT_GO: '1'
+          TULDOK_NATIVE_RETENTION_DIR: ${{ github.workspace }}/build/qa/external-project/native-retained
+        run: |
+          build/workspace-training/bin/python - <<'PY'
+          import os, sys, unittest
+          import numpy, sklearn, psycopg, torch
+          assert os.environ['TULDOK_TRAINING_PYTHON'] == sys.executable
+          names = [
+              'test_project_native_run.NativeRunSaveAsTests.test_one_actual_native_run_save_as_cold_role_and_metadata_only_git_clone',
+              'test_project_native_study.NativeStudyCustodyTests.test_actual_native_study_spent_budget_and_cold_history_require_fresh_admission',
+              'test_project_native_study.NativeStudyCustodyTests.test_native_study_admission_canonical_ack_then_sql_failure_never_starts_training',
+          ]
+          loader = unittest.TestLoader()
+          suite = loader.loadTestsFromNames(names)
+          assert not loader.errors and suite.countTestCases() == len(names)
+          result = unittest.TextTestRunner(verbosity=2).run(suite)
+          sys.exit(0 if result.wasSuccessful() and result.testsRun == len(names) and not result.skipped else 1)
+          PY
       - name: Create isolated optional agent verification environment
         run: python -m venv build/workspace-agent
       - run: build/workspace-agent/bin/python -m pip install -r requirements.txt
@@ -88,7 +145,111 @@ jobs:
       - run: node --check static/experiment-lineage.js
       - run: node tests/test_experiment_lineage_controller.cjs
       - run: build/workspace-agent/bin/python -m unittest discover -s tests -p test_artifact_segments.py
+      - name: Verify real external-project SDK CLI MCP and current authority
+        env:
+          PYTHONPATH: tests
+        run: |
+          build/workspace-agent/bin/python - <<'PY'
+          import mcp, sys, unittest
+          loader = unittest.TestLoader()
+          suite = loader.loadTestsFromName('test_project_files_m2_http.ProjectFilesM2HTTP.test_real_sdk_cli_mcp_roles_lost_reply_and_stale_binding')
+          assert not loader.errors and suite.countTestCases() == 1
+          result = unittest.TextTestRunner(verbosity=2).run(suite)
+          sys.exit(0 if result.wasSuccessful() and result.testsRun == 1 and not result.skipped else 1)
+          PY
+      - name: Verify independent fork and extraction across actual owner transports
+        env:
+          PYTHONPATH: tests
+        run: |
+          build/workspace-agent/bin/python - <<'PY'
+          import mcp, sys, unittest
+          names = [
+              'test_project_files_m4_clients_http.ProjectFilesM4ClientsHTTP.test_exact_file_revisions_save_as_and_current_role_through_sdk_cli_mcp',
+              'test_project_transfer_clients_http.ProjectTransferClientsHTTP.test_owner_handoff_and_single_extraction_through_sdk_cli_and_real_mcp',
+          ]
+          loader = unittest.TestLoader()
+          suite = loader.loadTestsFromNames(names)
+          assert not loader.errors and suite.countTestCases() == len(names)
+          result = unittest.TextTestRunner(verbosity=2).run(suite)
+          sys.exit(0 if result.wasSuccessful() and result.testsRun == len(names) and not result.skipped else 1)
+          PY
+      - name: Verify finite owner MCP modes and denial boundaries
+        env:
+          PYTHONPATH: tests
+        run: |
+          build/workspace-agent/bin/python - <<'PY'
+          import mcp, sys, unittest
+          loader = unittest.TestLoader()
+          suite = loader.loadTestsFromName('test_project_files_owner_mcp')
+          selected = suite.countTestCases()
+          assert not loader.errors and selected > 0
+          result = unittest.TextTestRunner(verbosity=2).run(suite)
+          sys.exit(0 if result.wasSuccessful() and result.testsRun == selected and not result.skipped else 1)
+          PY
+      - name: Verify real native project SDK CLI MCP and zero-GO recovery
+        env:
+          PYTHONPATH: tests
+          TULDOK_NATIVE_PROJECT_GO: '1'
+        run: |
+          build/workspace-agent/bin/python - <<'PY'
+          import mcp, sys, unittest
+          loader = unittest.TestLoader()
+          suite = loader.loadTestsFromName('test_project_native_clients_http.NativePublicClientsHTTP.test_real_native_preflight_replay_zero_go_failure_and_original_phase_recovery')
+          assert not loader.errors and suite.countTestCases() == 1
+          result = unittest.TextTestRunner(verbosity=2).run(suite)
+          sys.exit(0 if result.wasSuccessful() and result.testsRun == 1 and not result.skipped else 1)
+          PY
+      - name: Verify explicit note deletion history and cold fork across owner transports
+        env:
+          PYTHONPATH: tests
+          TULDOK_NOTE_DELETION_GO: '1'
+        run: |
+          build/workspace-agent/bin/python - <<'PY'
+          import mcp, sys, unittest
+          loader = unittest.TestLoader()
+          suite = loader.loadTestsFromName('test_project_file_deletion_clients_http.NoteDeletionClientsHTTP.test_real_note_deletion_restore_history_and_cold_independent_fork')
+          assert not loader.errors and suite.countTestCases() == 1
+          result = unittest.TextTestRunner(verbosity=2).run(suite)
+          sys.exit(0 if result.wasSuccessful() and result.testsRun == 1 and not result.skipped else 1)
+          PY
       - run: python -m unittest discover -s tests
+      - name: Verify external-project authoring and cold reconstruction in Chromium
+        run: node tests/browser_project_files_m2.cjs
+        env:
+          BROWSER: /usr/bin/google-chrome
+      - name: Verify named independent fork and inert cold clone in Chromium
+        run: node tests/browser_project_files_m4.cjs
+        env:
+          BROWSER: /usr/bin/google-chrome
+      - name: Verify explicit runtime failure recovery and inert cold Open in Chromium
+        run: node tests/browser_project_files_m3.cjs
+        env:
+          BROWSER: /usr/bin/google-chrome
+      - name: Verify source stop and independent extraction handoff in Chromium
+        run: node tests/test_workspace_project_transfer.cjs --browser
+        env:
+          BROWSER: /usr/bin/google-chrome
+          TULDOK_M5_UI_BROWSER_GO: '1'
+      - name: Verify named native preflight and original-phase recovery in Chromium
+        run: node tests/browser_project_files_native.cjs
+        env:
+          BROWSER: /usr/bin/google-chrome
+          TULDOK_NATIVE_PROJECT_GO: '1'
+      - name: Verify explicit note deletion restore and inert cold fork in Chromium
+        run: node tests/browser_project_file_deletion.cjs
+        env:
+          BROWSER: /usr/bin/google-chrome
+          TULDOK_NOTE_DELETION_GO: '1'
+      - name: Verify external-project working file and recovery controls
+        run: |
+          node tests/test_workspace_project_files_m4.cjs
+          node tests/test_workspace_project_fork_text.cjs
+          node tests/test_workspace_project_runtime.cjs
+          node tests/test_workspace_project_candidate.cjs
+          node tests/test_workspace_project_transfer.cjs
+          node tests/test_workspace_project_native.cjs
+          node tests/test_workspace_project_native_validation.cjs
+          node tests/test_workspace_project_file_deletion.cjs
       - run: node --check static/auth.js
       - run: node tests/test_auth_helpers.cjs
       - run: node tests/test_auth_controller.cjs

access.py

@@ -2,7 +2,7 @@
 import argparse
 from contextlib import contextmanager
 from contextvars import ContextVar
-from dataclasses import dataclass
+from dataclasses import dataclass, replace
 from functools import wraps
 import hashlib
 import json
@@ -24,6 +24,11 @@
 HASH = re.compile(r'^[a-f0-9]{64}$')
 
 
+def current_context():
+    """Read the immutable caller context; this observation grants no authority."""
+    return _CURRENT.get()
+
+
 class AccessError(Exception):
     def __init__(self, message, code='forbidden', status=403):
         super().__init__(message)
@@ -79,11 +84,29 @@ def guarded(self, *args, **kwargs):
 
 
 class Access:
-    def __init__(self, db, catalog_path):
+    def __init__(self, db, catalog_path, authority=None):
         self.db, self.catalog_path = db, Path(catalog_path)
         self.catalog_id, self.lock = db.dataset_id, db.lock
+        # Only current host credentials cross this bridge. Job admissions,
+        # execution grants and their consumption remain in this local projection.
+        if authority is not None and (not isinstance(authority, Access) or authority.authority is not None):
+            raise ValueError('A project bridge requires a direct host authority.')
+        self.authority = authority
+
+    def _host_authentication(self):
+        if self.authority is not None:
+            raise AccessError('Authenticate and administer credentials through the current host.', 'forbidden', 403)
+
+    def actor_exists(self, actor_id):
+        """Current host actor membership, never an imported scientific identity."""
+        if self.authority is not None:
+            return self.authority.actor_exists(actor_id)
+        with self.lock:
+            return self.db.execute('SELECT id FROM access_actors WHERE id=%s', (actor_id,)).fetchone() is not None
 
     def _credential(self, credential_id):
+        if self.authority is not None:
+            return self.authority._credential(credential_id)
         row = self.db.execute('''SELECT c.id,c.actor_id,c.kind AS credential_kind,c.scopes_json,c.expires_at,
             c.revoked,c.consumed,a.kind,a.label FROM access_credentials c
             JOIN access_actors a ON a.id=c.actor_id WHERE c.id=%s''', (credential_id,)).fetchone()
@@ -92,6 +115,7 @@ def _credential(self, credential_id):
         return dict(row)
 
     def _resolve(self, token, *, login=False):
+        self._host_authentication()
         if type(token) is not str or not 32 <= len(token) <= 256 or not token.isascii():
             raise AccessError('A valid credential is required.', 'unauthenticated', 401)
         row = self.db.execute('SELECT id,actor_id,kind,expires_at,revoked,consumed FROM access_credentials WHERE token_sha256=%s',
@@ -104,6 +128,7 @@ def _resolve(self, token, *, login=False):
 
     @contextmanager
     def bind(self, token):
+        self._host_authentication()
         try:
             with self.lock:
                 row = self._resolve(token)
@@ -121,6 +146,19 @@ def _context(self):
         context = _CURRENT.get()
         if context is None:
             raise AccessError('An explicit actor context is required.', 'unauthenticated', 401)
+        if self.authority is not None and self.catalog_id == self.authority.catalog_id:
+            raise AccessError('This project identity collides with the host authority. Save an independent project before execution.', 'unavailable', 409)
+        if self.authority is not None and context.mode not in ('credential','job','system'):
+            raise AccessError('Unsupported project actor context.')
+        if self.authority is not None and context.mode == 'credential':
+            if context.catalog_id != self.authority.catalog_id:
+                raise AccessError('Project execution requires a current host credential context.')
+            # Recheck both credential identity and liveness, including for APIs
+            # that use _context directly rather than require().
+            row = self.authority._credential(context.credential_id)
+            if row['actor_id'] != context.actor_id:
+                raise AccessError('Credential actor identity changed.')
+            return replace(context, catalog_id=self.catalog_id)
         if context.catalog_id != self.catalog_id:
             raise AccessError('Actor context belongs to another catalog.')
         return context
@@ -223,6 +261,7 @@ def denial(self, error, action, target=None, *, anonymous=False):
         error.audited = True
 
     def _mint(self, actor_id, kind, scopes, ttl):
+        self._host_authentication()
         token = secrets.token_urlsafe(32)
         credential_id = uuid.uuid4().hex
         self.db.execute('INSERT INTO access_credentials VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)',
@@ -232,6 +271,7 @@ def _mint(self, actor_id, kind, scopes, ttl):
 
     def _local_owner_token(self, *, initial=False, require_existing=False):
         """Explicit local administration only; never exposed by a transport route."""
+        self._host_authentication()
         with self.lock, self.db.transaction():
             owners = self.db.execute("SELECT id FROM access_actors WHERE kind='owner'").fetchall()
             if require_existing and not owners:
@@ -250,6 +290,7 @@ def _local_owner_token(self, *, initial=False, require_existing=False):
             return token
 
     def exchange(self, token):
+        self._host_authentication()
         try:
             with self.lock, self.db.transaction():
                 row = self._resolve(token, login=True)
@@ -270,6 +311,7 @@ def exchange(self, token):
 
     @authorized('admin', 'access.issue', owner_only=True)
     def issue(self, body):
+        self._host_authentication()
         if type(body) is not dict or set(body) - {'label', 'scopes', 'ttl_seconds'} or not {'label', 'scopes'} <= set(body):
             raise AccessError('Credential fields must be label, scopes and optional ttl_seconds.', 'invalid', 400)
         label = _text(body['label'], 'Label')
@@ -293,6 +335,7 @@ def access(self):
 
     @authorized('admin', 'access.revoke', owner_only=True)
     def revoke(self, credential_id):
+        self._host_authentication()
         with self.lock, self.db.transaction():
             changed = self.db.execute('UPDATE access_credentials SET revoked=1 WHERE id=%s RETURNING id', (credential_id,)).fetchone()
             if changed is None:
@@ -302,6 +345,7 @@ def revoke(self, credential_id):
         return {'revoked': True, 'credential_id': credential_id}
 
     def logout(self):
+        self._host_authentication()
         context = self.require('inspect', 'access.logout')
         if context.mode != 'credential':
             raise AccessError('Only a credential session can log out.')
@@ -366,7 +410,7 @@ def grant_execution(self, material_sha256, consumer_actor_id, ttl_seconds=600):
             raise AccessError('Approval lifetime must be 60 through 3600 seconds.', 'invalid', 400)
         with self.lock, self.db.transaction():
             context = self.require('approve-execution', 'access.grant_execution', owner_only=True)
-            if self.db.execute('SELECT id FROM access_actors WHERE id=%s', (consumer_actor_id,)).fetchone() is None:
+            if not self.actor_exists(consumer_actor_id):
                 raise AccessError('Consuming actor does not belong to this catalog.', 'unavailable', 404)
             created = int(time.time())
             grant = dict(id=uuid.uuid4().hex, catalog_id=self.catalog_id,